diff --git a/.github/instructions/api.instructions.md b/.github/instructions/api.instructions.md index e29553d9b..a19e68521 100644 --- a/.github/instructions/api.instructions.md +++ b/.github/instructions/api.instructions.md @@ -94,10 +94,12 @@ async def publish_score_set( worker: ArqRedis = Depends(get_worker), ): # ... validation and DB updates ... - await worker.enqueue_job( - "create_variants_for_score_set", - score_set.id, - correlation_id, + await enqueue_pipeline_for_score_set( + db=db, + redis=worker, + pipeline_name="publish_score_set", + score_set=item, + user=user_data.user, ) ``` @@ -114,24 +116,20 @@ responses=shared_responses # Defines 4xx/5xx response schemas Most write operations trigger a multi-step pipeline via the worker: ```python -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set # In a router endpoint: -pipeline, entrypoint_job_run = PipelineFactory.create_pipeline( +pipeline, job = await enqueue_pipeline_for_score_set( db=db, - name="validate_map_annotate_score_set", - pipeline_params={ - "score_set_id": score_set.id, - "updater_id": user_data.user.id, - "correlation_id": logging_context().get("correlation_id"), - }, + redis=worker, + pipeline_name="validate_map_annotate_score_set", + score_set=item, + user=user_data.user, + extra_params={"scores_file_key": scores_file_key, ...}, ) -db.commit() - -await worker.enqueue_job("start_pipeline", entrypoint_job_run.id) ``` -This creates a `Pipeline` with multiple `JobRun` records and `JobDependency` records, then enqueues the pipeline's `start_pipeline` entrypoint in ARQ. The worker coordinates the rest — each job runs after its dependencies complete. +`enqueue_pipeline_for_score_set` is the single kickoff seam shared by routers and scripts. It fills `score_set_id`, `updater_id` and `correlation_id`, uses `PipelineFactory` to create a `Pipeline` with its `JobRun` and `JobDependency` records, then enqueues the pipeline's `start_pipeline` entrypoint in ARQ. A failed enqueue discards the just-created pipeline records. The worker coordinates the rest — each job runs after its dependencies complete. ### Job Function Signature @@ -155,18 +153,11 @@ Callers pass only `ctx` and `job_id` when enqueueing. The decorator creates the Correlation IDs flow from the API request through the pipeline to each job: ```python -# In the router — capture correlation ID from starlette-context -from mavedb.lib.logging.context import save_to_logging_context, logging_context +# In the router — save request context; the kickoff seam reads the correlation ID from it +from mavedb.lib.logging.context import save_to_logging_context save_to_logging_context({"score_set_urn": urn}) -correlation_id = logging_context().get("correlation_id") - -# Pass to pipeline via pipeline_params -pipeline, entrypoint = PipelineFactory.create_pipeline( - db=db, - name="validate_map_annotate_score_set", - pipeline_params={"correlation_id": correlation_id, ...}, -) +await enqueue_pipeline_for_score_set(db=db, redis=worker, pipeline_name="validate_map_annotate_score_set", ...) ``` Each job retrieves the correlation ID from its `job_params` and uses `job_manager.save_to_context()` for structured logging. diff --git a/.github/instructions/copilot-instructions.md b/.github/instructions/copilot-instructions.md index 089220c66..884e83245 100644 --- a/.github/instructions/copilot-instructions.md +++ b/.github/instructions/copilot-instructions.md @@ -12,8 +12,8 @@ MaveDB is a bioinformatics database API for Multiplex Assays of Variant Effect ( ### Service Architecture - **FastAPI application** — `src/mavedb/server_main.py`, router-based endpoint organization - **Background worker** — ARQ/Redis for async processing (variant creation, mapping, publication, annotation) -- **Multi-container setup** — API server, worker, PostgreSQL, Redis, plus external services (cdot-rest, dcd-mapping, seqrepo) -- **Docker config** — `docker-compose-dev.yml` (6 services), multi-stage `Dockerfile` +- **Multi-container setup** — API server, worker, PostgreSQL 15, Redis, moto (local S3), plus external services (cdot-rest, dcd-mapping, seqrepo) +- **Docker config** — `docker-compose-dev.yml` (9 services), multi-stage `Dockerfile` ### Core Domain Model - **Hierarchical URN system**: ExperimentSet (`urn:mavedb:00000001`) → Experiment (`00000001-a`) → ScoreSet (`00000001-a-1`) → Variant (ScoreSet URN + `#` + number) @@ -41,8 +41,8 @@ src/mavedb/ ├── worker/ # ARQ background worker system │ ├── jobs/ # Job function implementations (by category) │ │ ├── registry.py # Central registry of all jobs, cron definitions -│ │ ├── variant_processing/ # Variant creation and mapping -│ │ ├── external_services/ # ClinGen, ClinVar, gnomAD, UniProt +│ │ ├── variant_processing/ # Variant creation, mapping, reverse translation +│ │ ├── external_services/ # ClinGen, ClinVar, gnomAD, UniProt, VEP │ │ ├── pipeline_management/ # Pipeline entrypoint (start_pipeline) │ │ └── system/ # Cron maintenance (cleanup stalled jobs) │ ├── lib/ # Infrastructure layer @@ -143,7 +143,7 @@ poetry run python -m mavedb.scripts. - [score_set.py](src/mavedb/models/score_set.py) — Primary data model patterns - [score_sets.py](src/mavedb/routers/score_sets.py) — Complex router with worker integration -- [jobs.py](src/mavedb/worker/jobs.py) — Background processing patterns +- [registry.py](src/mavedb/worker/jobs/registry.py) — Background job registration - [score_set.py](src/mavedb/view_models/score_set.py) — Pydantic model hierarchy - [server_main.py](src/mavedb/server_main.py) — App setup and dependency injection - [authentication.py](src/mavedb/lib/authentication.py) — Auth patterns @@ -222,9 +222,10 @@ poetry run python -m mavedb.scripts. - **Key job types**: - `create_variants_for_score_set` - Parse uploaded CSV, create variant records - `map_variants_for_score_set` - Map variants via DCD Mapping / VRS + - `reverse_translate_variants_for_score_set` - Derive equivalent nucleotide/protein alleles and projection groups via variant-annotation - `submit_score_set_mappings_to_car/ldh` - Submit to ClinGen services - `cleanup_stalled_jobs` - Cron job for recovering stuck jobs -- **Enqueueing pipelines**: Routers call `PipelineFactory.create_pipeline()` then `ArqRedis.enqueue_job("start_pipeline", ...)` with the pipeline's entrypoint JobRun ID +- **Enqueueing pipelines**: Routers and scripts call `enqueue_pipeline_for_score_set()` (`src/mavedb/lib/workflow/kickoff.py`), which creates the pipeline via `PipelineFactory.create_pipeline()` and enqueues its `start_pipeline` entrypoint JobRun - **Detailed documentation**: See `src/mavedb/worker/README.md` and `.github/instructions/worker.instructions.md` ### View Models (Pydantic) diff --git a/.github/instructions/database.instructions.md b/.github/instructions/database.instructions.md index f2b3de6d1..947fc36cc 100644 --- a/.github/instructions/database.instructions.md +++ b/.github/instructions/database.instructions.md @@ -8,13 +8,12 @@ applyTo: 'src/mavedb/models/**/*.py,alembic/**/*.py' ## Model Structure ### Base Class -All models use `@as_declarative()` with a shared `Base` class providing: -- `id`: Integer primary key (auto-increment) -- `creation_date` / `modification_date`: Auto-managed timestamps -- `created_by_id` / `modified_by_id`: Foreign keys to users table +All models subclass the `@as_declarative()` `Base` in `src/mavedb/db/base.py`. It declares no columns; it only defaults `__tablename__` to the lowercased class name. Table models declare their own `id` primary key and set `__tablename__` explicitly; view models map `__table__` to a view instead. ### Common Columns Most resource models include: +- `creation_date` / `modification_date`: `Date` columns defaulting to today (`modification_date` updates on write) +- `created_by_id` / `modified_by_id`: Foreign keys to the users table (user-owned resources: experiment sets, experiments, score sets, calibrations, calibration controls, collections) - `urn`: Unique string identifier (temporary `tmp:` or permanent `urn:mavedb:...`) - `private`: Boolean flag for visibility control - `approved`: Boolean for moderation status @@ -50,6 +49,16 @@ class ScoreSetPublicationIdentifierAssociation(Base): primary = Column(Boolean, default=False) ``` +### Valid-Time Versioning (`ValidTime`) +Rows that change over time mix in `ValidTime` (`src/mavedb/db/mixins.py`) instead of being updated or deleted in place: +- `valid_from` / `valid_to`: a row is live while `valid_to` is NULL; history is the half-open window `[valid_from, valid_to)`. +- Reads: filter explicitly with `Model.current` or `Model.live_at(as_of)` (`None` means current; also works on `aliased()` classes). Never assume a query returns only live rows. +- Replacing a row: `supersede_with` (one row) or `supersede_live_where` (bulk) retire the old rows and stamp the new rows' `valid_from` with the same timestamp, so the handoff has no gap. Never retire and then insert separately. +- Withdrawing with no successor: `retire` (one row) or `retire_live_where` (bulk UPDATE). +- `__retire_cascade__`: relationship names whose live `ValidTime` children are retired when the parent is (one level deep). `supersede_live_where` raises on a class that declares it; use `supersede_with` per row. +- Each table adds a partial unique index over its natural key `WHERE valid_to IS NULL`, so a second live row raises `IntegrityError`. +- Content-addressed, immutable rows (e.g. alleles) do not use the mixin. + ### Enum Columns Use non-native enums stored as strings: ```python @@ -121,4 +130,4 @@ if item.private and not user_has_access(user, item): ``` ## Materialized Views -Used for pre-computed aggregations. Refresh via worker cron jobs defined in `src/mavedb/worker/settings.py`. +Used for pre-computed aggregations. Refresh via worker cron jobs defined in `src/mavedb/worker/jobs/registry.py`. diff --git a/.github/instructions/worker.instructions.md b/.github/instructions/worker.instructions.md index 16b6e19f0..8d654c2d5 100644 --- a/.github/instructions/worker.instructions.md +++ b/.github/instructions/worker.instructions.md @@ -86,7 +86,7 @@ Always return using factory methods: ```python return JobExecutionOutcome.succeeded(data={"variants_created": count}) -return JobExecutionOutcome.failed(reason="No mapped variants found", data={...}) +return JobExecutionOutcome.failed(reason="No mapping records found", data={...}) return JobExecutionOutcome.failed(reason="HGVS parse error", failure_category=FailureCategory.DATA_ERROR) return JobExecutionOutcome.skipped(data={"reason": "Feature disabled"}) # For unhandled exceptions: let them propagate — the decorator catches and creates .errored() @@ -124,11 +124,12 @@ Parameters with `None` values in pipeline definitions are filled at runtime from - **Slack safety**: `send_slack_job_failure()` and `send_slack_job_error()` catch their own exceptions internally (logging critical on failure), so Slack outages never interfere with job lifecycle management or error recovery in the decorators. - **Stale RUNNING recovery**: `start_job()` accepts RUNNING as a startable status (alongside QUEUED and PENDING). When ARQ re-delivers a job after a worker crash, `start_job()` logs a warning and resets the start timestamp rather than raising `JobTransitionError`. - **Concurrency limit**: `max_jobs = 2` in `ArqWorkerSettings` prevents event loop starvation from sync psycopg2 DB calls. With the default `max_jobs=10`, multiple concurrent jobs issuing blocking DB operations can starve the asyncio event loop. +- **Fast lane**: `start_pipeline` and `create_variants_for_score_set` run on a separate queue (`FAST_LANE_QUEUE_NAME`) consumed by a second worker process (`ArqFastLaneWorkerSettings`), so long mapping jobs can't block new pipelines. Every `enqueue_job` and every `arq.jobs.Job(...)` status/abort call must pass `_queue_name=arq_queue_name(job_run)`. ## Pipeline Lifecycle (Brief) -1. Router calls `PipelineFactory.create_pipeline()` → creates `Pipeline`, `JobRun`, and `JobDependency` records -2. Router enqueues the `start_pipeline` entrypoint job in ARQ +1. Router or script calls `enqueue_pipeline_for_score_set()` (`lib/workflow/kickoff.py`) → `PipelineFactory.create_pipeline()` creates `Pipeline`, `JobRun`, and `JobDependency` records +2. The same call enqueues the `start_pipeline` entrypoint job in ARQ 3. `start_pipeline` runs → its `@with_pipeline_management` decorator starts the pipeline and calls `coordinate_pipeline()` 4. `coordinate_pipeline()` finds PENDING jobs whose dependencies are met → marks them QUEUED → enqueues in ARQ 5. Each job runs → after completion, its decorator calls `coordinate_pipeline()` again @@ -181,7 +182,7 @@ Pipeline definitions live in `src/mavedb/lib/workflow/definitions.py`. The `Pipe | `lib/managers/pipeline_manager.py` | Pipeline coordination, dependency resolution, job enqueueing | | `lib/managers/constants.py` | Status groupings (`TERMINAL_JOB_STATUSES`, `STARTABLE_JOB_STATUSES`, etc.) | | `lib/managers/exceptions.py` | Exception hierarchy (`JobStateError`, `PipelineCoordinationError`, etc.) | -| `settings/worker.py` | `ArqWorkerSettings` class (ARQ worker configuration) | +| `settings/worker.py` | `ArqWorkerSettings` (main) and `ArqFastLaneWorkerSettings` classes (ARQ worker configuration) | | `settings/lifecycle.py` | Worker startup/shutdown hooks, `standalone_ctx()` | | `src/mavedb/lib/workflow/definitions.py` | Pipeline and job definitions (`PIPELINE_DEFINITIONS`) | | `src/mavedb/lib/workflow/pipeline_factory.py` | Creates Pipeline + JobRun + JobDependency records | diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..16f8319bc --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +VENV := .venv/bin + +.DEFAULT_GOAL := help + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}' + +.PHONY: dev +dev: ## Install deps including local editable variant-annotation + poetry install --extras server + +.PHONY: test +test: ## Run the test suite + $(VENV)/pytest tests/ + +.PHONY: lint +lint: ## Check code with ruff + $(VENV)/ruff check src/ tests/ + +.PHONY: format +format: ## Format code with ruff + $(VENV)/ruff format src/ tests/ diff --git a/README.md b/README.md index e59bbc13e..8155e470c 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ To stop the application when it is running as a daemon, run docker-compose -f docker-compose-local.yml down ``` -`docker-compose-local.yml` configures four containers: one for the API server, one for the PostgreSQL database, one for the -worker node and one for the Redis cache which acts as the job queue for the worker node. The worker node stores data in a Docker -volume named `mavedb-redis` and the database stores data in a Docker volume named `mavedb-data`. Both these volumes will persist +`docker-compose-local.yml` configures five containers: one for the API server, one for the PostgreSQL 15 database, two worker +nodes and one for the Redis cache which acts as the job queue for the workers. The fast worker runs only pipeline starts and +variant creation, on its own queue, so new score sets get variants while mapping jobs occupy the main worker. The worker node stores data in a Docker +volume named `mavedb-redis` and the database stores data in a Docker volume named `mavedb-data-15`. Both these volumes will persist after running `docker-compose down`. **Notes** @@ -88,6 +89,21 @@ after running `docker-compose down`. The database username and password should be edited for production deployments. `NCBI_API_KEY` will be removed in the future. **TODO** Move these to an .env file. +2. Optional database and CSV export tuning variables. Times are in seconds, and setting a timeout or `CSV_BUILD_SLOTS` + to `0` disables it. All of them apply only to the API server; the worker, scripts and migrations run without them. + + | Variable | Default | Purpose | + |---|---|---| + | `DB_POOL_SIZE` / `DB_MAX_OVERFLOW` | 5 / 10 | Persistent and burst connections per process | + | `DB_POOL_TIMEOUT` | 10 | Longest wait for a free pooled connection before the request returns 503 | + | `DB_STATEMENT_TIMEOUT_SECONDS` | 30 | Longest single statement before the request returns 503 | + | `DB_CSV_STATEMENT_TIMEOUT_SECONDS` | 45 | The same limit for the score set CSV routes | + | `DB_LOCK_TIMEOUT_SECONDS` | 5 | Longest wait for a row or table lock before the request returns 503 | + | `DB_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS` | 60 | Longest idle gap inside an open transaction before Postgres ends the session | + | `DB_SLOW_STATEMENT_SECONDS` | 5 | Statements slower than this are logged with their route | + | `CSV_BUILD_SLOTS` | 2 | Score set CSV builds of `CSV_BUILD_MIN_ROWS` or more rows each process runs at once | + | `CSV_BUILD_MIN_ROWS` | 10000 | Smaller builds skip the cap. A large build waits up to 10s for a slot, then returns 503 with `Retry-After` | + ### Running the API server in Docker for development A similar procedure can be followed to run the API server in development mode on your local machine. There are a couple diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py new file mode 100644 index 000000000..3d2de474d --- /dev/null +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -0,0 +1,1433 @@ +"""Backfill the Allele substrate (``mapping_records`` / ``alleles`` / +``mapping_record_alleles`` + the annotation link tables) from existing +``mapped_variants`` rows. + +Context +------- +The annotation/serving layer is migrating off ``MappedVariant`` onto a +parallel-tables model:: + + Variant ─< MappingRecord (1 live/variant, ValidTime) ─< MappingRecordAllele (ValidTime) >─ Allele (dedup by vrs_digest) + +This script populates the new tables from the *existing* ``mapped_variants`` data so +the read-cutover (the ``v_variant_annotations`` view / ``published_variants`` MV +rewrites and the serving readers) resolves for historical score sets, which are +otherwise empty in the new substrate. + +This is the **reshape half of the hybrid migration**: it reconstructs the +authoritative spine deterministically from old data with no external-service +calls. It deliberately does **not** reproduce the reverse-translation +equivalence fan-out (the per-level coding/genomic/protein alleles and their +``projection_group`` pairing) — that data was never computed for old mappings +(``MappedVariant`` holds a single ``post_mapped`` VRS plus cross-level HGVS +*strings*, and an Allele needs a real ``vrs_digest`` that a string cannot +supply). Restoring that richness is the decoupled *enrichment* half: queue the +reverse-translation + annotation-refresh jobs per score set on the worker after +this backfill lands. Serving is correct at the measured level as soon as this +runs; enrichment only adds cross-level breadth and blocks nothing. + +What is reconstructed +--------------------- +* **MappingRecord history** — every ``MappedVariant`` version of a variant with a resolvable sequence + level becomes a ``MappingRecord`` with a gap-free ``[valid_from, valid_to)`` window derived from the + versions' ``mapped_date`` ordering. The ``current=True`` version is the live tail (``valid_to IS + NULL``). History dates are inherently approximate: ``MappedVariant`` records versioning with a + ``current`` boolean, not transaction time, so a window boundary is the successor's ``mapped_date``, + not the exact instant the swap happened. A version whose sequence level can't be attributed is + skipped **individually** (its siblings still get their records — see Idempotency below); if that + version happens to be the live one, the variant has no live record and stays eligible for retry. + Live dates that precede a historical version's are a logged anomaly (a likely re-map candidate), not + a fatal condition — see :func:`_reconstruct_windows`. +* **Authoritative Allele** — from each version's ``post_mapped`` VRS, deduplicated by ``vrs_digest`` + against an in-memory, per-chunk cache (:func:`_authoritative_allele` — not the live job's + ``get_or_create_allele``, which queries per call; see :func:`do_migration`'s docstring for why). + Carries the CAID and the assay-level HGVS + across so the allele-keyed serving endpoints resolve. The assay-level HGVS itself is read through a + three-tier fallback (:func:`_hgvs_assay_level_with_source`): ``mv.hgvs_assay_level`` directly, else + the per-level column matching the resolved level (``hgvs_g``/``hgvs_c``/``hgvs_p`` — backfilled later + and never caught up on a large share of old data), else the ``post_mapped`` VRS blob's own + ``expressions`` (the mapper sometimes stamps or reconstructs an HGVS expression directly on the VRS + Allele even when neither dedicated column got populated). Even with all three tiers, some legacy rows + carry no recoverable HGVS at all — that gap is real and needs the mapper re-run to close, not another + migration-time trick. Reading only ``hgvs_assay_level`` (as this script originally did) left most + reconstructed Alleles (and ``MappingRecord.hgvs_assay_level``, and therefore its derived + ``transcript``) NULL even when a real string was recoverable — silently starving any downstream pass + that resolves a transcript from it. +* **MappingRecordAllele** — one ``is_authoritative=True`` link per record on the + record's validity window. ``projection_group`` stays NULL; ``mavedb.scripts.project_measured_variants`` + writes the measured projection group afterwards. +* **Annotation timelines** — reconstructed across *every* mapping version, not just the live one: + ``gnomad_variants`` → ``gnomad_allele_links``, ``clinical_controls`` → ``clinvar_allele_links``, + VEP columns → ``vep_allele_consequences``. Each mapping version's bundled annotations are captured + against that version's window and, per allele, merged per source key (gnomAD variant / ClinVar + control / consequence value) into maximal intervals — so a continuously-held annotation is one link + dated from its **earliest** mapping version, open-ended only if it reaches the live record. A + release change that coincided with a re-map surfaces as a superseding closed+live pair. This is + faithful (the old jobs wrote annotations on the current mapping row, so their changes aligned to + re-map boundaries), just **coarse** — sub-mapping-version churn collapses to the latest value in + that window, and boundary timestamps are day-grained. The valid-time axis is uniformly the mapping + date across sources (VEP's real ``access_date`` is kept in its own audit column, not as + ``valid_from``, so ``as_of`` stays coherent). VEP has no Ensembl release string in old data, but + ``access_date`` is enough to derive one: ``source_version`` is looked up from a snapshotted table of + Ensembl release dates (:data:`_ENSEMBL_VEP_RELEASE_DATES`), not stamped as an opaque sentinel — see + :func:`_resolve_vep_source_version`. Only a missing ``access_date``, or one older than the table's + earliest known release, falls back to the ``"legacy"`` sentinel. + + **Exception:** a gnomAD or ClinVar window that would otherwise land live on a **protein-level** allele + is force-closed at the migration's own run time instead (VEP is unaffected — see below). The old + model could proxy both onto protein-level rows because there was nowhere else to attach them; the new + model resolves them correctly through the CA/PA link graph onto nucleotide siblings instead + (``lib/allele_measurements.py``), so carrying the proxy forward live would just create a second, + competing provenance story for the same value. Applied uniformly even though gnomAD is not known to + have actually been proxied this way in practice (ClinVar was) — it is the same underlying anti-pattern + (a nucleotide-level concept attached to a protein-level allele), so the guard costs nothing to keep + general. The historical fact is kept (closed, not deleted) for point-in-time audit; live annotations + for these alleles return once cross-level annotation fan-out re-derives them properly. + +Idempotency / no double-write +----------------------------- +A variant is either old-substrate or new (the parallel-tables invariant); native new-model variants +have no ``mapped_variants``. The migration therefore processes only variants that have +``mapped_variants`` and **no live** ``MappingRecord`` yet — live, not "any", because a variant whose +live mapped_variant version has no resolvable sequence level is left with only historical records (see +above) and must stay selectable so a later re-map or QC fix can complete it. Within a selected variant, +each window is additionally guarded by a ``(variant_id, mapped_date)`` existence check, so a re-run +that revisits a partially-migrated variant only fills in the windows still missing rather than +duplicating the ones a prior pass already wrote. Annotation links are existence-checked before +insertion, so a partial run resumes cleanly; run ``rebuild-annotations`` after resuming a +partially-migrated variant to pick up any timelines the skipped windows didn't get to build. + +Standalone convention +--------------------- +Follows the ``migrate_jsonb_ranges_to_table_rows.py`` pattern: invoked outside +``alembic upgrade`` (it can take a long time on production data), commits in +variant batches, and offers ``verify`` / ``rollback`` commands. + +Usage +----- +:: + + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate # run + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate verify # inspect without writing + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rollback # remove backfilled records (destructive) + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rebuild-annotations # (re)build annotation links for already-migrated variants + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --score-set urn:mavedb:00000001-a-1 + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --batch-size 1000 --dry-run +""" + +from __future__ import annotations + +import argparse +import logging +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import date, datetime, time, timezone +from time import perf_counter # `time` above is datetime.time; import the timer directly +from typing import Optional + +import sqlalchemy as sa +from biocommons.seqrepo import SeqRepo +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, configure_mappers + +from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import — register all mappers for configure_mappers() +from mavedb.db.session import SessionLocal +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from pydantic import ValidationError + +from mavedb.data_providers.services import seqrepo +from mavedb.lib.hgvs import extract_accession +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget +from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.lib.vrs_utils import CANONICAL_ACCESSION_PREFIXES, canonical_variation_document, location_refgets +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +configure_mappers() + +logger = logging.getLogger(__name__) + +# Old VEP data carries no Ensembl release string, only an access_date — but source_version is derivable: +# Ensembl releases (and VEP's version with them) are coordinated, dated, and public +# (https://github.com/Ensembl/ensembl-vep/releases), so the release active on a given access_date can +# be looked up rather than stamped as an opaque sentinel. Each entry is a release's *earliest* GitHub +# tag date (patch tags like 92.1-92.6 are hotfixes within release 92, not new majors); an access_date +# is attributed to the newest release whose date is <= it. Snapshotted from the GitHub API on +# 2026-07-20 (covers releases 92 through 116); this table does not self-update, so re-run the same +# query and extend it before backfilling data annotated under a release newer than the last entry here. +_ENSEMBL_VEP_RELEASE_DATES: list[tuple[date, str]] = [ + (date(2018, 4, 12), "92"), + (date(2018, 7, 17), "93"), + (date(2018, 10, 2), "94"), + (date(2019, 1, 10), "95"), + (date(2019, 4, 8), "96"), + (date(2019, 7, 3), "97"), + (date(2019, 9, 27), "98"), + (date(2020, 1, 16), "99"), + (date(2020, 4, 29), "100"), + (date(2020, 8, 20), "101"), + (date(2020, 11, 30), "102"), + (date(2021, 2, 16), "103"), + (date(2021, 5, 5), "104"), + (date(2021, 12, 9), "105"), + (date(2022, 4, 14), "106"), + (date(2022, 7, 13), "107"), + (date(2022, 10, 20), "108"), + (date(2023, 2, 8), "109"), + (date(2023, 7, 18), "110"), + (date(2024, 1, 11), "111"), + (date(2024, 5, 16), "112"), + (date(2024, 10, 18), "113"), + (date(2025, 5, 8), "114"), + (date(2025, 9, 3), "115"), + (date(2026, 6, 10), "116"), +] + +# Fallback sentinel for the cases the release-date table can't resolve: no access_date at all, or an +# access_date older than the table's earliest known release (GitHub has no tags before release/92, so +# anything before 2018-04-12 is out of range). VepAlleleConsequence.source_version is NOT NULL, and a +# refresh run should treat this value as unversioned and re-confirm it against the current release +# rather than trusting it as current — the same behavior the sentinel always had. +LEGACY_VEP_SOURCE_VERSION = "legacy" + + +def _resolve_vep_source_version(access_date: Optional[date]) -> str: + """Map a legacy VEP ``access_date`` to the Ensembl release active on that day. + + Returns :data:`LEGACY_VEP_SOURCE_VERSION` when ``access_date`` is missing or predates the earliest + release the table knows about — both cases where the release genuinely can't be determined, as + opposed to just not being in the table's covered range for lack of maintenance. + """ + if access_date is None or access_date < _ENSEMBL_VEP_RELEASE_DATES[0][0]: + return LEGACY_VEP_SOURCE_VERSION + resolved = _ENSEMBL_VEP_RELEASE_DATES[0][1] + for release_date, version in _ENSEMBL_VEP_RELEASE_DATES: + if release_date > access_date: + break + resolved = version + return resolved + + +def _as_dt(d: date) -> datetime: + """Lift a ``date`` to a timezone-aware ``datetime`` for a ValidTime boundary. + + ValidTime windows are ``DateTime(timezone=True)``; ``MappedVariant`` versioning + is day-grained (``mapped_date``), so midnight UTC is the reconstructed boundary. + """ + return datetime.combine(d, time.min, tzinfo=timezone.utc) + + +def _resolve_level(mv: MappedVariant) -> Optional[SequenceLevel]: + """The sequence level a ``MappedVariant`` was assayed/aligned at. + + ``alignment_level`` is populated on every row by the target-gene-mapping QC + backfill and is the direct source (the mapping job sets a record's + ``assay_level`` and ``alignment_level`` to the same value). For the rare + legacy row the QC backfill could not attribute, fall back to whichever mapped + HGVS column is populated. Returns ``None`` when neither signal is present — + the record cannot be created (``assay_level`` is NOT NULL) and the caller + skips the variant with a logged anomaly. + """ + if mv.alignment_level is not None: + return mv.alignment_level + if mv.hgvs_g: + return SequenceLevel.genomic + if mv.hgvs_c: + return SequenceLevel.cdna + if mv.hgvs_p: + return SequenceLevel.protein + return None + + +_LEVEL_HGVS_COLUMN = { + SequenceLevel.genomic: "hgvs_g", + SequenceLevel.cdna: "hgvs_c", + SequenceLevel.protein: "hgvs_p", +} + + +def _hgvs_assay_level_with_source(mv: MappedVariant, level: SequenceLevel) -> tuple[Optional[str], str]: + """Resolve the assay-level HGVS string, reporting which of three tiers supplied it. + + ``mv.hgvs_assay_level`` is frequently ``NULL`` on legacy rows — it was backfilled later and never + caught up on a large share of old data — and reading only it (as this script originally did) meant + most reconstructed Alleles got a NULL ``hgvs_g``/``hgvs_c``/``hgvs_p`` and + ``MappingRecord.hgvs_assay_level`` even when the real string was recoverable. Since + ``MappingRecord.transcript`` derives from ``hgvs_assay_level``, that silently starved any downstream + pass that resolves a transcript from it (e.g. cross-level annotation fan-out). + + Three tiers, in order: + + 1. ``"direct"`` — ``mv.hgvs_assay_level`` itself. + 2. ``"level_column"`` — the per-level column matching ``level`` (``hgvs_g``/``hgvs_c``/``hgvs_p``), + mirroring :func:`_resolve_level`'s own fallback (same column), just for the *value* rather than + the level. + 3. ``"vrs_expression"`` — the ``post_mapped`` VRS blob's own ``expressions``. Some legacy rows have + *neither* dedicated column populated, but the mapper stamped an HGVS expression directly onto + the VRS Allele it produced (``dcd_mapping`` sets this unconditionally for coding-layer alleles, + and reconstructs it for genomic/protein when accession resolution succeeds — so it is not + universal either, just an independent, additional chance). Reuses + :func:`mavedb.lib.variants.get_hgvs_from_post_mapped`, the same extraction serving already relies + on, rather than re-parsing VRS JSON here. A malformed or VRS-1.x blob raises there; caught here + and treated as "no expression" so one bad blob doesn't abort the whole variant. + + Returns ``(None, "none")`` when no tier resolves anything — some legacy data genuinely carries no + recoverable HGVS at all, and that gap cannot be closed without re-running the mapper. + """ + if mv.hgvs_assay_level: + return mv.hgvs_assay_level, "direct" + + level_value = getattr(mv, _LEVEL_HGVS_COLUMN[level]) + if level_value: + return level_value, "level_column" + + try: + vrs_value = get_hgvs_from_post_mapped(mv.post_mapped) + except (KeyError, ValueError): + vrs_value = None + if vrs_value: + return vrs_value, "vrs_expression" + + return None, "none" + + +def _resolve_hgvs_assay_level(mv: MappedVariant, level: SequenceLevel) -> Optional[str]: + """The assay-level HGVS string for a version — see :func:`_hgvs_assay_level_with_source`.""" + return _hgvs_assay_level_with_source(mv, level)[0] + + +@dataclass +class _WindowResult: + """``_reconstruct_windows``'s output: the ordered windows plus an anomaly flag. + + Kept as one return value (rather than ``None``-means-skip on windows, anomaly bundled in) so a + caller can't forget to check the flag the way a bare tuple invites. + """ + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] + live_precedes_history: bool + + +def _reconstruct_windows(mvs: list[MappedVariant]) -> Optional[_WindowResult]: + """Order a variant's ``MappedVariant`` versions and assign ValidTime windows. + + Returns ``(mv, valid_from, valid_to)`` per version, gap-free: each version's + ``valid_to`` is its successor's ``valid_from``, and the live tail's is + ``None``. The ``current=True`` version is the live tail regardless of its + ``mapped_date``. Returns ``None`` (skip) when no version is current — a + variant with mappings but no live one is a source anomaly we do not guess a + live state for. + + ``live_precedes_history=True`` flags the case where the live version's ``mapped_date`` is earlier + than some historical version's — a data-quality anomaly (``current`` should always be the newest). + When it happens, the pairwise clamp below still prevents inverted windows, but at the cost of + collapsing the affected historical window to zero-width (invisible to ``as_of``) rather than + losing data outright: the caller surfaces the flag as a loud, variant-attributed warning instead + of silently reordering, since it likely means the source variant wants a re-map. + """ + current = [m for m in mvs if m.current] + history = [m for m in mvs if not m.current] + + if not current: + return None + if len(current) == 1: + live = current[0] + else: + # Multiple live rows is a source anomaly; keep the newest live and demote + # the rest into history so no data is dropped. + current.sort(key=lambda m: (m.mapped_date, m.id)) + live = current[-1] + history.extend(current[:-1]) + logger.warning("Variant %s has %d current mapped_variants; keeping newest live.", live.variant_id, len(current)) + + history.sort(key=lambda m: (m.mapped_date, m.id)) + ordered = history + [live] + live_precedes_history = bool(history) and live.mapped_date < history[-1].mapped_date + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] = [] + for i, m in enumerate(ordered): + valid_from = _as_dt(m.mapped_date) + valid_to: Optional[datetime] = None + if i + 1 < len(ordered): + successor = _as_dt(ordered[i + 1].mapped_date) + # Same-day or out-of-order successor: clamp so the window is never + # inverted (a zero-width [t, t) window is simply invisible to as_of). + valid_to = max(successor, valid_from) + windows.append((m, valid_from, valid_to)) + return _WindowResult(windows=windows, live_precedes_history=live_precedes_history) + + +def _fill_missing_allele_fields(allele: Allele, mv: MappedVariant, level: SequenceLevel) -> None: + """Backfill identity fields onto a (possibly pre-existing, deduped) allele. + + Dedup means the first variant to mint an allele sets its fields; a later + variant sharing the ``vrs_digest`` may carry the CAID or level HGVS the first + lacked. Those are content-derived (deterministic from the allele), so + fill-if-missing is safe and never overwrites a populated value. + """ + if allele.clingen_allele_id is None and mv.clingen_allele_id: + allele.clingen_allele_id = mv.clingen_allele_id + + hgvs_col = _LEVEL_HGVS_COLUMN[level] + hgvs_value = _resolve_hgvs_assay_level(mv, level) + if getattr(allele, hgvs_col) is None and hgvs_value: + setattr(allele, hgvs_col, hgvs_value) + + +class _SequenceGuard: + """Checks that an allele sits on the sequence SeqRepo holds for its HGVS accession. + + The backfill counterpart of :func:`mavedb.lib.vrs_utils.verify_allele_refget`, which the mapping and + reverse translation jobs run at insert time. Legacy rows can put one accession on two sequences + (``NP_001346945.1`` on two refgets), and ``uq_alleles_hgvs`` lets only one of + them hold each HGVS. Without this check, whichever the backfill reaches first wins. SeqRepo's answer + is cached per accession for the run. + """ + + def __init__(self, sr: SeqRepo) -> None: + self._sr = sr + self._expected: dict[str, Optional[str]] = {} + self.mismatches = 0 + + def agrees(self, variation: dict, hgvs: Optional[str]) -> bool: + """Return False if the variation's refget differs from SeqRepo's for the accession. + + An accession SeqRepo can't resolve passes, as it does in ``verify_allele_refget``: there is nothing + to compare against. + """ + accession = extract_accession(hgvs or "") + if not accession.startswith(CANONICAL_ACCESSION_PREFIXES): + return True + if accession not in self._expected: + try: + self._expected[accession] = resolve_refget(self._sr, accession) + except (SequenceNotFoundError, AmbiguousSequenceError) as e: + logger.error(f"Cannot verify refgets on {accession}: {e}") + self._expected[accession] = None + expected = self._expected[accession] + if expected is None or all(refget == expected for refget in location_refgets(variation)): + return True + self.mismatches += 1 + return False + + +def _authoritative_allele( + db: Session, + mv: MappedVariant, + level: SequenceLevel, + allele_cache: dict[str, Allele], + sequence_guard: _SequenceGuard, +) -> Optional[Allele]: + """Get-or-create the authoritative allele from ``mv.post_mapped``, via an in-memory dedup cache. + + Returns ``None`` for versions with no post-mapped representation (failed or benign-absent variants + get a record but no linked allele, matching the mapping job), and for versions on a sequence other + than SeqRepo's for their accession (see :class:`_SequenceGuard`); the enrich re-map rebuilds those. + + Unlike :func:`mavedb.lib.variant_translations.get_or_create_allele` (used by the live mapping job), + this does **not** query the database or flush per call. ``allele_cache`` is pre-seeded by the + caller with one bulk ``vrs_digest IN (...)`` query per chunk (see :func:`do_migration`), and this + function keeps it updated in-memory as new alleles are drafted — so a digest shared by two windows + in the *same* chunk (a common dedup case) resolves from the dict, not a second round trip. This is + safe only because the session runs with ``autoflush=False`` (a query wouldn't see an unflushed + insert anyway) and because the caller flushes once per variant, not per window: an actual + ``uq_alleles_vrs_digest`` collision (e.g. a concurrent live mapping run) still surfaces at that + flush, inside the same per-variant SAVEPOINT it always has — this only removes the redundant reads, + not the write-time integrity check. + """ + stored = mv.post_mapped or {} + if not stored.get("id"): + return None + + # Recompute identity rather than adopt the stored id. `vrs_digest` is the UNIQUE dedup key the + # whole allele graph hangs off, `alleles` rows are immutable once written, and `alleles` is the + # only table here with no ValidTime - so a wrong digest cannot be corrected in place or + # reconstructed by `as_of`, and it silently collapses two different alleles into one row. + # Historical `post_mapped` blobs cannot be trusted on this point: an id minted before VRS + # normalization moved a del/dup span survives unchanged, because `ga4gh_identify`'s default + # `in_place` returns a non-empty id untouched. The live mapping job canonicalizes for the same + # reason (`worker/jobs/variant_processing/mapping.py`); a backfill that trusted the blob would + # write drift into the substrate permanently. See `scripts/audit_allele_identifiers.py`. + try: + post_mapped, digest = canonical_variation_document(stored, subject=f"mapped_variant {mv.id}") + except (ValidationError, ValueError): + logger.exception(f"Could not canonicalize post_mapped for mapped_variant {mv.id}; skipping its allele.") + return None + + hgvs_value = _resolve_hgvs_assay_level(mv, level) + if not sequence_guard.agrees(post_mapped, hgvs_value): + logger.warning(f"mapped_variant {mv.id} ({hgvs_value}) is not on SeqRepo's sequence; skipping its allele.") + return None + + existing = allele_cache.get(digest) + if existing is None: + # `allele_cache` is seeded by a per-chunk prefetch keyed on the STORED `post_mapped` id, but + # alleles deduplicate by their RECOMPUTED digest. Under VRS id drift (stored id != recomputed + # digest) an already-persisted allele carrying this true digest is absent from the cache, so a + # blind INSERT below would violate `uq_alleles_vrs_digest`, and the failed flush would roll the + # whole variant back — dropping its MappingRecord, not just the allele link. Fall back to a + # lookup by the recomputed digest (only on a cache miss, i.e. the drifted minority) so a match + # deduplicates onto the existing row instead of colliding. See scripts/audit_allele_identifiers. + existing = db.scalar(sa.select(Allele).where(Allele.vrs_digest == digest)) + if existing is not None: + allele_cache[digest] = existing + if existing is not None: + _fill_missing_allele_fields(existing, mv, level) + return existing + + allele = Allele( + vrs_digest=digest, + level=level, + hgvs_g=hgvs_value if level == SequenceLevel.genomic else None, + hgvs_c=hgvs_value if level == SequenceLevel.cdna else None, + hgvs_p=hgvs_value if level == SequenceLevel.protein else None, + clingen_allele_id=mv.clingen_allele_id, + post_mapped=post_mapped, + ) + db.add(allele) + allele_cache[digest] = allele + return allele + + +@dataclass +class _AnnotationObservation: + """One mapping version's annotation payload for an authoritative allele, tagged with that + version's validity window. + + Built fresh from the DB per allele-batch (:func:`_observations_for_alleles`) rather than collected + inline during record creation — so a deduped allele shared by several variants, or spanning a + variant's re-map history, still yields one merged timeline rather than per-version fragments (see + :func:`_write_annotation_timelines`), without needing every observation for the whole run held in + memory at once. + """ + + valid_from: datetime + valid_to: Optional[datetime] + gnomad_variant_ids: tuple[int, ...] + clinvar_control_ids: tuple[int, ...] + vep_consequence: Optional[str] + vep_access_date: Optional[date] + + +def _observation_for( + mv: MappedVariant, valid_from: datetime, valid_to: Optional[datetime] +) -> Optional[_AnnotationObservation]: + """Capture a mapping version's annotations for its window, or ``None`` if it carries none.""" + gnomad_ids = tuple(g.id for g in mv.gnomad_variants) + clinvar_ids = tuple(c.id for c in mv.clinical_controls) + if not gnomad_ids and not clinvar_ids and not mv.vep_functional_consequence: + return None + return _AnnotationObservation( + valid_from=valid_from, + valid_to=valid_to, + gnomad_variant_ids=gnomad_ids, + clinvar_control_ids=clinvar_ids, + vep_consequence=mv.vep_functional_consequence, + vep_access_date=mv.vep_access_date, + ) + + +def _merge_intervals( + windows: list[tuple[datetime, Optional[datetime]]], +) -> list[tuple[datetime, Optional[datetime]]]: + """Merge overlapping/adjacent ``[valid_from, valid_to)`` windows; ``valid_to=None`` is open-ended. + + An annotation held across a run of mapping versions produces contiguous windows (and overlapping + ones when a deduped allele is shared by several variants). Merging collapses each continuous run + into one link — ``valid_from`` = its earliest mapping date, ``valid_to`` open only if the run + reaches the live record — and closes a link only where the annotation actually lapses, rather than + fabricating a boundary at every re-map. + """ + ordered = sorted(windows, key=lambda w: w[0]) + merged: list[tuple[datetime, Optional[datetime]]] = [] + cur_from, cur_to = ordered[0] + for vf, vt in ordered[1:]: + if cur_to is None or vf <= cur_to: # overlap/adjacency; an open-ended window absorbs the rest + cur_to = None if (cur_to is None or vt is None) else max(cur_to, vt) + else: + merged.append((cur_from, cur_to)) + cur_from, cur_to = vf, vt + merged.append((cur_from, cur_to)) + return merged + + +def _allele_has_live_link(db: Session, model: type, allele_id: int) -> bool: + """Whether ``allele_id`` already has a live (``valid_to IS NULL``) link in ``model``. + + Sees **committed** rows only — the session runs ``autoflush=False``, so this does not observe + not-yet-flushed adds from the current pass. It therefore catches prior-run and concurrently- + committed live links (e.g. from the mapping pipeline running alongside this backfill); single-live + within one pass is enforced separately, in memory, by :func:`_write_allele_timeline`. + """ + return db.scalar(sa.select(model.id).where(model.allele_id == allele_id, model.valid_to.is_(None))) is not None + + +def _emit_gnomad_link( + db: Session, + allele_id: int, + gnomad_variant_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(GnomadAlleleLink.id).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.gnomad_variant_id == gnomad_variant_id, + GnomadAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + db.add( + GnomadAlleleLink( + allele_id=allele_id, gnomad_variant_id=gnomad_variant_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["gnomad_links"] += 1 + + +def _emit_clinvar_link( + db: Session, + allele_id: int, + clinvar_control_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(ClinvarAlleleLink.id).where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.clinvar_control_id == clinvar_control_id, + ClinvarAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run; ClinVar is multi-live per (allele, control) + return + db.add( + ClinvarAlleleLink( + allele_id=allele_id, clinvar_control_id=clinvar_control_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["clinvar_links"] += 1 + + +def _emit_vep_consequence( + db: Session, + allele_id: int, + consequence: str, + access_date: date, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(VepAlleleConsequence.id).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.functional_consequence == consequence, + VepAlleleConsequence.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + source_version = _resolve_vep_source_version(access_date) + stats[ + "vep_source_version_legacy_fallback" + if source_version == LEGACY_VEP_SOURCE_VERSION + else "vep_source_version_resolved" + ] += 1 + db.add( + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + source_version=source_version, + access_date=access_date, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["vep_consequences"] += 1 + + +def _write_allele_timeline( + db: Session, allele_id: int, obs_list: list[_AnnotationObservation], stats: Counter, run_at: datetime +) -> None: + """Build and emit one allele's merged annotation link timelines (no flush/commit here). + + Keyed per source — gnomAD by ``gnomad_variant_id``, ClinVar by ``clinvar_control_id``, VEP by + consequence value — each key's windows are merged so a continuously-held annotation is a single + link dated from its earliest mapping version, open-ended only if it reaches the live record. + + **Single-live-per-allele** for gnomAD/VEP is enforced with an in-memory flag seeded from committed + state (:func:`_allele_has_live_link`). This is the fix for the ``autoflush=False`` session: a + deduped allele linked to two *different* gnomAD variants (from two variants sharing the allele) + would otherwise emit two live links and violate ``uq_gnomad_allele_links_live`` at flush, since the + per-add DB check can't see the earlier unflushed add. The flag also folds in a link already live + from a prior run or the concurrent mapping pipeline. ClinVar is multi-live per ``(allele, control)``. + + **Protein-level gnomAD/ClinVar are a legacy proxy the new model doesn't carry forward live.** The + old model could attach ClinVar significance and gnomAD frequency to protein-level ``MappedVariant`` + rows as a proxy — there was nowhere else to put them. Both are intrinsically nucleotide-level + concepts (a ClinVar submission and a gnomAD allele frequency are both keyed to a genomic variant, + not a protein change), and the new model resolves them correctly for a protein allele via the CA/PA + link graph onto its nucleotide siblings (see ``lib/allele_measurements.py``), not by direct + attachment. A live direct link here would just be a second, competing provenance story for the same + value. So a protein allele's gnomAD/ClinVar window is force-closed at ``run_at`` (the migration's + own run time) instead of staying open-ended: the historical fact that this had been proxy-annotated + is kept for point-in-time audit, but it stops being *live* data at the moment of reshape, to be + replaced once cross-level annotation fan-out re-derives it correctly through the link graph. VEP is + exempt — a functional consequence is computed relative to the assayed representation itself, so it + is not a proxied nucleotide-level value the way gnomAD/ClinVar are. + """ + gnomad_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + clinvar_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_windows: dict[str, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_access: dict[str, list[date]] = defaultdict(list) + + for obs in obs_list: + for gnomad_variant_id in obs.gnomad_variant_ids: + gnomad_windows[gnomad_variant_id].append((obs.valid_from, obs.valid_to)) + for clinvar_control_id in obs.clinvar_control_ids: + clinvar_windows[clinvar_control_id].append((obs.valid_from, obs.valid_to)) + if obs.vep_consequence: + vep_windows[obs.vep_consequence].append((obs.valid_from, obs.valid_to)) + if obs.vep_access_date is not None: + vep_access[obs.vep_consequence].append(obs.vep_access_date) + + is_protein = False + if gnomad_windows or clinvar_windows: + allele_level = db.scalar(sa.select(Allele.level).where(Allele.id == allele_id)) + is_protein = allele_level == SequenceLevel.protein.value + + gnomad_live_taken = _allele_has_live_link(db, GnomadAlleleLink, allele_id) + for gnomad_variant_id, windows in gnomad_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + # Force-closed, not left live: not subject to the single-live guard below, since it + # never becomes a live row that could collide with uq_gnomad_allele_links_live. + valid_to = run_at + stats["protein_gnomad_links_closed"] += 1 + elif valid_to is None: + if gnomad_live_taken: + logger.warning( + "Allele %s already has a live gnomAD link; skipping gnomad_variant %s.", + allele_id, + gnomad_variant_id, + ) + continue + gnomad_live_taken = True + _emit_gnomad_link(db, allele_id, gnomad_variant_id, valid_from, valid_to, stats) + + for clinvar_control_id, windows in clinvar_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + valid_to = run_at + stats["protein_clinvar_links_closed"] += 1 + _emit_clinvar_link(db, allele_id, clinvar_control_id, valid_from, valid_to, stats) + + vep_live_taken = _allele_has_live_link(db, VepAlleleConsequence, allele_id) + for consequence, windows in vep_windows.items(): + observed_dates = vep_access[consequence] + for valid_from, valid_to in _merge_intervals(windows): + if valid_to is None: + if vep_live_taken: + logger.warning("Allele %s already has a live VEP consequence; skipping %r.", allele_id, consequence) + continue + vep_live_taken = True + access_date = min(observed_dates) if observed_dates else valid_from.date() + _emit_vep_consequence(db, allele_id, consequence, access_date, valid_from, valid_to, stats) + + +def _write_annotation_timelines( + db: Session, observations: dict[int, list[_AnnotationObservation]], stats: Counter, run_at: datetime +) -> None: + """Write each allele's annotation timelines, isolating conflicts per allele. + + Each allele is built + flushed inside its own SAVEPOINT: if a live-uniqueness collision still + slips through the in-memory guard — e.g. the concurrently-running mapping pipeline commits a live + link for a shared allele between the guard's committed-state read and this flush — only that + allele's links roll back (counted as ``annotation_conflicts``), and the rest of the pass proceeds + instead of the whole final commit aborting. + + ``run_at`` is a single timestamp for the whole pass (not re-read per allele), so every protein-level + ClinVar link force-closed by :func:`_write_allele_timeline` in this run shares the same cutover + instant rather than drifting with wall-clock time across a long batch. + """ + for allele_id, obs_list in observations.items(): + try: + with db.begin_nested(): + _write_allele_timeline(db, allele_id, obs_list, stats, run_at) + db.flush() # surface any live-uniqueness violation inside this allele's savepoint + except IntegrityError: + stats["annotation_conflicts"] += 1 + logger.warning("Annotation links for allele %s conflicted (live elsewhere); skipped.", allele_id) + + +def _observations_for_alleles(db: Session, allele_ids: list[int]) -> dict[int, list[_AnnotationObservation]]: + """Rebuild annotation observations for one batch of alleles, from fresh DB state. + + Scoped by allele id, not by variant/chunk: the merge :func:`_write_allele_timeline` performs is only + ever correct once *every* mapping version that ever authoritatively linked to a given allele has been + observed, and alleles are shared arbitrarily across variants — a variant-id chunk boundary says + nothing about which alleles are "done" (an allele's contributing variants can land in different + chunks). Batching by allele id instead means every batch's observations are complete for the alleles + in it, however many separate variant-chunks originally fed those alleles. + + Pairs each authoritative ``MappingRecord`` to its source ``MappedVariant`` version by + ``(variant_id, mapped_date)`` — the record was created with that version's ``mapped_date`` — mirroring + the pairing :func:`_migrate_variant` used when the record was first built. + """ + links = db.execute( + sa.select( + MappingRecordAllele.allele_id, + MappingRecord.variant_id, + MappingRecord.mapped_date, + MappingRecord.valid_from, + MappingRecord.valid_to, + ) + .join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .where(MappingRecordAllele.allele_id.in_(allele_ids), MappingRecordAllele.is_authoritative) + ).all() + if not links: + return {} + + variant_ids = {variant_id for _, variant_id, _, _, _ in links} + mv_by_key: dict[tuple[int, date], MappedVariant] = { + (mv.variant_id, mv.mapped_date): mv + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(variant_ids))).all() + } + + observations: dict[int, list[_AnnotationObservation]] = defaultdict(list) + for allele_id, variant_id, mapped_date, valid_from, valid_to in links: + mv = mv_by_key.get((variant_id, mapped_date)) + if mv is None: + continue # authoritative link survives independently of its source mapped_variant row + observation = _observation_for(mv, valid_from, valid_to) + if observation is not None: + observations[allele_id].append(observation) + return observations + + +def _backfill_annotation_timelines( + db: Session, allele_ids: list[int], *, batch_size: int, run_at: datetime, stats: Counter +) -> None: + """Build and commit annotation link timelines for ``allele_ids``, batch by batch. + + Each batch re-reads its own observations fresh from the DB (:func:`_observations_for_alleles`) and + commits before moving to the next, so memory is bounded by ``batch_size`` alleles at a time + regardless of how many millions of variants fed into ``allele_ids``. This replaces the previous + design, which accumulated every ``_AnnotationObservation`` for the *whole run* in memory before + writing anything at the very end — correct, but the accumulator's unbounded growth over a + multi-million-variant run put steadily increasing pressure on the GC, degrading throughput well + before the run completed (not just a flat per-call cost, but one that compounded with how much had + already been processed). + """ + total = len(allele_ids) + pass_start = perf_counter() + for offset in range(0, total, batch_size): + chunk = allele_ids[offset : offset + batch_size] + observations = _observations_for_alleles(db, chunk) + if observations: + _write_annotation_timelines(db, observations, stats, run_at) + db.commit() + processed = min(offset + batch_size, total) + elapsed = perf_counter() - pass_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Built timelines for {processed}/{total} allele(s)... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + +def _migrate_variant( + db: Session, + variant_id: int, + score_set_id: int, + mvs: list[MappedVariant], + stats: Counter, + skipped: dict[str, list[int]], + annotation_manager: AnnotationStatusManager, + existing_records: set[tuple[int, date]], + allele_cache: dict[str, Allele], + sequence_guard: _SequenceGuard, +) -> set[int]: + """Reconstruct as much of a variant's record history as its data supports, returning the ids of + every allele this variant authoritatively touched. + + Also records one ``VRS_MAPPING`` event per version created (variant-subject) and one + ``CLINGEN_ALLELE_ID`` / ``MAPPED_HGVS`` event per allele-touch (allele-subject), all reason + :data:`EventReason.MIGRATED` — see :func:`do_migration`'s docstring for why this is scoped to the + mapping/identity side only, not gnomAD/ClinVar/VEP. + + The caller only accumulates these bare ids (not the annotation payload itself) into a global, + whole-run set — a dedup key is all a deferred, allele-batched annotation pass + (:func:`_backfill_annotation_timelines`) needs to know which alleles to revisit; the actual + gnomAD/ClinVar/VEP observations are re-read fresh from the DB per allele-batch there, not carried + in memory from here. Carrying full ``_AnnotationObservation`` payloads for the whole run was the + previous design and is what overloaded the heap at scale — see that function's docstring. Returns + an empty set for a variant with no current version at all — that one is a full skip, since there is + no live state to reconstruct. + + A version with no resolvable sequence level is skipped **individually**, not the whole variant: + ``ValidTime`` history is exactly the mechanism for retaining partial provenance, so one bad version + should not cost its siblings their records. A window's ``valid_from``/``valid_to`` only depend on + its neighbors' ``mapped_date`` (computed up front in :func:`_reconstruct_windows`), never on + whether a record actually got created for those neighbors — so skipping a window here doesn't + distort the boundaries around it. Already-migrated windows (from a prior partial run that + stopped here before this fix, or a live-level failure that keeps this variant eligible for retry — + see :func:`_variant_ids_to_migrate`) are detected via ``existing_records`` (bulk pre-fetched once per + chunk by the caller — see :func:`do_migration`) and skipped as a no-op, so a re-run is safe to + resume mid-variant. Their annotations are not re-derived here; run ``rebuild-annotations`` after a + resumed run to backfill any timelines missed by the skip. + + Every ``MappingRecord``/``Allele``/``MappingRecordAllele`` this variant needs is only ``db.add``'d + here, not flushed, until one ``db.flush()`` at the end covering the whole variant — cut from one + flush per *window* (this was the dominant cost at scale: round trips, not query time, were the + bottleneck). ``MappingRecordAllele`` is built from the ``record``/``allele`` *objects*, not their + ``.id``, specifically so it doesn't need them flushed first — SQLAlchemy resolves the FKs at flush + time. Observations and ``MIGRATED`` events (which need real ``allele.id``/``variant_id`` values) are + built in a second pass over ``pending`` *after* that single flush, once ids exist. + + Skips/anomalies are recorded into ``skipped`` (reason → variant ids), not logged per variant: a + single unmappable score set (e.g. a large pairwise-haplotype library the mapper never converted) + can contribute hundreds of thousands of skips, so :func:`do_migration` rolls them up into one line + per score set at the end rather than emitting a warning each. + """ + result = _reconstruct_windows(mvs) + if result is None: + stats["variants_skipped_no_current"] += 1 + skipped["no_current"].append(variant_id) + logger.debug("Variant %s has mapped_variants but none current; skipped.", variant_id) + return set() + + if result.live_precedes_history: + # Not fatal — the clamp in _reconstruct_windows keeps windows non-inverted — but it silently + # collapses a historical window to zero-width, so surface it loudly: this shape means the + # live mapped_variant's mapped_date is older than a superseded one, which shouldn't happen + # absent a source data-quality issue, and likely wants a re-map to fix at the source. + stats["variants_live_precedes_history"] += 1 + skipped["live_precedes_history"].append(variant_id) + + # (mv, record, allele-or-None, hgvs_assay_level, valid_from, valid_to) per window actually created + # this pass — resolved to real ids by one flush below, then walked again to emit events and collect + # touched allele ids. + pending: list[ + tuple[MappedVariant, MappingRecord, Optional[Allele], Optional[str], datetime, Optional[datetime]] + ] = [] + + for mv, valid_from, valid_to in result.windows: + if (variant_id, mv.mapped_date) in existing_records: + continue # already migrated in a prior pass over this variant; resuming, not redoing + + level = _resolve_level(mv) + if level is None: + is_live = valid_to is None + reason = "no_level_live" if is_live else "no_level_historical" + stats[f"windows_skipped_{reason}"] += 1 + skipped[reason].append(variant_id) + logger.debug( + "Variant %s has an unattributable mapped_variant (id=%s, live=%s); window skipped.", + variant_id, + mv.id, + is_live, + ) + continue + + hgvs_assay_level, hgvs_source = _hgvs_assay_level_with_source(mv, level) + if hgvs_source == "level_column": + stats["hgvs_recovered_from_level_column"] += 1 + elif hgvs_source == "vrs_expression": + stats["hgvs_recovered_from_vrs_expression"] += 1 + + # Same treatment for the pre-mapped representation. MappingRecord.vrs_digest is not a dedup + # key, so a wrong value here is less destructive than on Allele - but it is still an identity + # claim about content this migration is republishing, and the drift has the same cause. + pre_mapped_document, pre_mapped_digest = None, None + if (mv.pre_mapped or {}).get("id"): + try: + pre_mapped_document, pre_mapped_digest = canonical_variation_document( + mv.pre_mapped, subject=f"mapped_variant {mv.id} pre_mapped" + ) + except (ValidationError, ValueError): + logger.exception(f"Could not canonicalize pre_mapped for mapped_variant {mv.id}; storing it unkeyed.") + pre_mapped_document = mv.pre_mapped + + record = MappingRecord( + variant_id=variant_id, + score_set_id=score_set_id, + vrs_digest=pre_mapped_digest, + pre_mapped=pre_mapped_document, + assay_level=level, + hgvs_assay_level=hgvs_assay_level, + mapped_date=mv.mapped_date, + vrs_version=mv.vrs_version, + mapping_api_version=mv.mapping_api_version, + target_gene_mapping_id=mv.target_gene_mapping_id, + alignment_level=mv.alignment_level, + at_mismatched_locus=mv.at_mismatched_locus, + near_gap=mv.near_gap, + valid_from=valid_from, + valid_to=valid_to, + ) + db.add(record) + stats["records"] += 1 + + allele = _authoritative_allele(db, mv, level, allele_cache, sequence_guard) + if allele is not None: + db.add( + MappingRecordAllele( + mapping_record=record, + allele=allele, + is_authoritative=True, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["record_alleles"] += 1 + + pending.append((mv, record, allele, hgvs_assay_level, valid_from, valid_to)) + + if not pending: + return set() + + db.flush() # one round trip resolves every record/allele/link id added for this whole variant + stats["variants"] += 1 + + touched_allele_ids: set[int] = set() + for mv, record, allele, hgvs_assay_level, valid_from, valid_to in pending: + # VRS_MAPPING: variant-subject, written for every version regardless of allele — mirrors the + # live job (mapping.py), whose FAILED/ABSENT outcomes get a record but no allele too. + if mv.error_message: + mapping_disposition = Disposition.FAILED + elif allele is not None: + mapping_disposition = Disposition.PRESENT + else: + mapping_disposition = Disposition.ABSENT + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant_id, + disposition=mapping_disposition, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 1 + + if allele is not None: + # CLINGEN_ALLELE_ID / MAPPED_HGVS: allele-subject, only meaningful once an allele exists. + # allele.id is real now — this loop runs after the single flush above. + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT if mv.clingen_allele_id else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + annotation_manager.record_event( + AnnotationType.MAPPED_HGVS, + allele_id=allele.id, + disposition=Disposition.PRESENT if hgvs_assay_level else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 2 + + touched_allele_ids.add(allele.id) + + # Flushed once for the whole variant (not per event) — still inside this variant's own SAVEPOINT + # (the caller's `with db.begin_nested():` wraps this entire call), so a buffered event never + # outlives the rollback of the record/allele it references. See do_migration's except-block for the + # backstop: if something *between* here and there still raised, pending events for this variant are + # explicitly discarded rather than leaking into a later, unrelated flush. + annotation_manager.flush() + return touched_allele_ids + + +def _variant_ids_to_migrate(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Variant ids that have ``mapped_variants`` but no *live* ``MappingRecord`` yet. + + Keyed on a live record specifically, not "any record at all": a variant can have historical + records from a prior pass but still be missing its live one (the live version's mapped_variant had + no resolvable sequence level — see :func:`_migrate_variant`), and that variant must stay eligible + for retry until a re-map (or a QC fix) lets its live window resolve. The parallel-tables invariant + still makes "has mapped_variants and no live mapping record" the exact set of variants with + incomplete substrate; native new-model variants have no ``mapped_variants``. + """ + has_live_record = sa.select(MappingRecord.id).where( + MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None) + ) + query = ( + sa.select(Variant.id) + .join(MappedVariant, MappedVariant.variant_id == Variant.id) + .where(~has_live_record.exists()) + ) + if score_set_urn is not None: + query = query.join(ScoreSet, ScoreSet.id == Variant.score_set_id).where(ScoreSet.urn == score_set_urn) + return list(db.scalars(query.distinct().order_by(Variant.id)).all()) + + +_SKIP_REASONS = { + "no_current": "mapped_variants present but none current", + "no_level_historical": "a superseded mapped_variant with no resolvable sequence level (window skipped, siblings rescued)", + "no_level_live": "the LIVE mapped_variant has no resolvable sequence level — no live record could be created; remains eligible for retry", + "live_precedes_history": "the live mapped_variant's mapped_date precedes a historical one (data anomaly — a re-map candidate)", +} + + +def _log_skip_summary(db: Session, skipped: dict[str, list[int]], *, chunk: int = 5000) -> None: + """Roll skipped variants up into one log line per (reason, score set). + + Skips cluster hard by score set — a whole library the mapper couldn't convert skips en masse — so + a per-variant warning drowns the log while a per-score-set count pinpoints exactly which set to + chase. Variant ids are resolved to score set URNs in chunks to keep the ``IN`` clauses bounded. + """ + for reason, variant_ids in skipped.items(): + if not variant_ids: + continue + counts: Counter = Counter() + for offset in range(0, len(variant_ids), chunk): + part = variant_ids[offset : offset + chunk] + for urn, n in db.execute( + sa.select(ScoreSet.urn, sa.func.count()) + .join(Variant, Variant.score_set_id == ScoreSet.id) + .where(Variant.id.in_(part)) + .group_by(ScoreSet.urn) + ).all(): + counts[urn] += n + detail = _SKIP_REASONS.get(reason, reason) + logger.warning("Skipped %d variant(s) across %d score set(s) — %s:", len(variant_ids), len(counts), detail) + for urn, n in counts.most_common(): + logger.warning(" %s: %d variant(s) skipped (%s).", urn, n, reason) + + +def do_migration( + db: Session, + *, + score_set_urn: Optional[str] = None, + batch_size: int = 1000, + dry_run: bool = False, + run_at: Optional[datetime] = None, +) -> None: + """Backfill the allele substrate from ``mapped_variants``, committing per batch. + + ``run_at`` (default: now) is the single cutover instant used to force-close any protein-level + ClinVar link that would otherwise land live — see :func:`_write_allele_timeline`. Exposed as a + parameter mainly for deterministic tests; a real run just wants "now". + + Also backfills the ``annotation_event`` audit log — but only for the mapping/identity side + (``VRS_MAPPING``, ``CLINGEN_ALLELE_ID``, ``MAPPED_HGVS``), not gnomAD/ClinVar/VEP. That scope split + is deliberate: the mapping side is a genuine, non-lossy reconstruction (one real historical event + per ``mapped_variants`` version, the same data already trusted enough to build ``MappingRecord``/ + ``Allele`` from), whereas gnomAD/ClinVar/VEP would need to guess at a created/reconfirmed/skipped + cadence legacy data never recorded — and those sources get real, fresh events again once each score + set is re-annotated, so the audit gap there is temporary, not permanent. See + :data:`EventReason.MIGRATED`. + """ + print("Backfilling the Allele substrate from mapped_variants...") + if dry_run: + print(" DRY RUN — no changes will be committed.") + run_at = run_at or datetime.now(timezone.utc) + annotation_manager = AnnotationStatusManager(db) + sequence_guard = _SequenceGuard(seqrepo()) + + variant_ids = _variant_ids_to_migrate(db, score_set_urn) + print(f"Found {len(variant_ids)} variants to migrate.") + + stats: Counter = Counter() + # Wall-clock spent in each phase, so a run reports where the time goes (batch build vs. the + # final annotation pass) and whether the per-batch rate degrades as the tables grow. + phase_seconds: Counter = Counter() + # Ids only — a deduped allele can be shared by variants in different chunks, so its annotation + # timeline can only be assembled once every contributing variant has been migrated, but the actual + # gnomAD/ClinVar/VEP payload doesn't need to ride along in memory to get there: it's re-read fresh + # from the DB per allele-batch in the deferred pass below (_backfill_annotation_timelines). Holding + # just ids here (not full _AnnotationObservation objects) keeps this set's footprint tiny even at + # millions of variants — see that function's docstring for why the old accumulate-everything design + # degraded badly at scale. + touched_allele_ids: set[int] = set() + # Skipped variant ids by reason, rolled up into a per-score-set summary at the end instead of a + # per-variant warning (one unmappable library can skip hundreds of thousands of variants). + skipped: dict[str, list[int]] = defaultdict(list) + + run_start = perf_counter() + for offset in range(0, len(variant_ids), batch_size): + chunk = variant_ids[offset : offset + batch_size] + + mark = perf_counter() + mvs_by_variant: dict[int, list[MappedVariant]] = {vid: [] for vid in chunk} + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(chunk))).all(): + mvs_by_variant[mv.variant_id].append(mv) + phase_seconds["fetch_mapped_variants"] += perf_counter() - mark + + # Bulk pre-fetch, once per chunk instead of once per window — this (not query execution time) + # was the actual bottleneck: a per-window round trip for "does this record already exist" plus + # another for "does this allele already exist" dominates wall-clock at millions-of-rows scale, + # far more than the cost of the queries themselves. + mark = perf_counter() + existing_records: set[tuple[int, date]] = set( + db.execute( + sa.select(MappingRecord.variant_id, MappingRecord.mapped_date).where( + MappingRecord.variant_id.in_(chunk) + ) + ).all() + ) + score_set_ids: dict[int, int] = dict( + db.execute(sa.select(Variant.id, Variant.score_set_id).where(Variant.id.in_(chunk))).tuples().all() + ) + candidate_digests = {(mv.post_mapped or {}).get("id") for mvs in mvs_by_variant.values() for mv in mvs} + candidate_digests.discard(None) + allele_cache: dict[str, Allele] = {} + if candidate_digests: + for allele in db.scalars(sa.select(Allele).where(Allele.vrs_digest.in_(candidate_digests))).all(): + allele_cache[allele.vrs_digest] = allele + phase_seconds["prefetch"] += perf_counter() - mark + + mark = perf_counter() + for variant_id in chunk: + # A per-variant SAVEPOINT so one bad variant rolls back only its own inserts (not the + # whole batch), and its touched allele ids are simply never added rather than referencing + # rolled-back rows. + pending_events_before = len(annotation_manager._pending) + try: + with db.begin_nested(): + variant_touched_allele_ids = _migrate_variant( + db, + variant_id, + score_set_ids[variant_id], + mvs_by_variant[variant_id], + stats, + skipped, + annotation_manager, + existing_records, + allele_cache, + sequence_guard, + ) + except Exception as exc: # noqa: BLE001 — one bad variant must not abort the batch + # Discard any events this variant buffered but never got to flush before raising — they + # would otherwise linger in the manager's queue and get inserted by a later, unrelated + # flush, still referencing this variant/allele's now-rolled-back id (an FK violation far + # from its actual cause). _migrate_variant flushes its own events before returning + # normally, so this is a no-op on the success path. + del annotation_manager._pending[pending_events_before:] + stats["variants_errored"] += 1 + logger.exception("Failed to migrate variant %s: %s", variant_id, exc) + continue + touched_allele_ids.update(variant_touched_allele_ids) + phase_seconds["build_records"] += perf_counter() - mark + + mark = perf_counter() + if dry_run: + db.rollback() + touched_allele_ids.clear() # nothing persisted this batch; these ids are now invalid + else: + db.commit() + phase_seconds["commit"] += perf_counter() - mark + + processed = min(offset + batch_size, len(variant_ids)) + elapsed = perf_counter() - run_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Processed {processed}/{len(variant_ids)} variants... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + if not dry_run and touched_allele_ids: + print(f"Building annotation timelines for {len(touched_allele_ids)} allele(s)...") + mark = perf_counter() + _backfill_annotation_timelines( + db, sorted(touched_allele_ids), batch_size=batch_size, run_at=run_at, stats=stats + ) + phase_seconds["annotation_timelines"] += perf_counter() - mark + + print("\nMigration completed:") + stats["alleles_off_sequence"] = sequence_guard.mismatches + for key in ( + "variants", + "records", + "record_alleles", + "alleles_off_sequence", + "hgvs_recovered_from_level_column", + "hgvs_recovered_from_vrs_expression", + "annotation_events_written", + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + "variants_skipped_no_current", + "windows_skipped_no_level_historical", + "windows_skipped_no_level_live", + "variants_live_precedes_history", + "variants_errored", + ): + print(f" {key}: {stats[key]}") + + _log_skip_summary(db, skipped) + + total = perf_counter() - run_start + print("\nTiming by phase:") + for phase in ("fetch_mapped_variants", "build_records", "commit", "annotation_timelines"): + seconds = phase_seconds[phase] + share = f"{100 * seconds / total:.0f}%" if total else "n/a" + print(f" {phase}: {seconds:.1f}s ({share})") + variants_done = stats["variants"] + overall_rate = f"{variants_done / total:.0f}/s" if total else "n/a" + print(f" total: {total:.1f}s ({overall_rate})") + + +def _allele_ids_to_rebuild(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Every allele id with an authoritative link, optionally scoped to one score set's variants.""" + query = sa.select(sa.distinct(MappingRecordAllele.allele_id)).where(MappingRecordAllele.is_authoritative) + if score_set_urn is not None: + query = ( + query.join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join(ScoreSet, ScoreSet.id == Variant.score_set_id) + .where(ScoreSet.urn == score_set_urn) + ) + return list(db.scalars(query.order_by(MappingRecordAllele.allele_id)).all()) + + +def rebuild_annotation_timelines( + db: Session, *, score_set_urn: Optional[str] = None, batch_size: int = 1000, run_at: Optional[datetime] = None +) -> None: + """(Re)build annotation link timelines for already-migrated variants, independently of record + creation. Idempotent (existence-checked, savepoint-isolated). Use to recover after a crash in the + ``migrate`` annotation pass, or to refresh timelines after fixing annotation logic. + + Rebuilds strictly from the legacy proxy data (``mapped_variants`` associations), same as + ``migrate`` — it is not the cross-level fan-out job — so it applies the same protein-level ClinVar + closure rule (see :func:`_write_allele_timeline`) using its own ``run_at`` (default: now). Batches by + allele id via :func:`_backfill_annotation_timelines`, the same bounded-memory pass ``migrate`` itself + uses — this used to accumulate every observation for the whole rebuild in memory before writing + anything, which had the identical unbounded-growth problem the ``migrate`` pass was fixed for. + """ + print("Rebuilding annotation timelines from existing records...") + run_at = run_at or datetime.now(timezone.utc) + allele_ids = _allele_ids_to_rebuild(db, score_set_urn) + print(f"Found {len(allele_ids)} allele(s) with authoritative links.") + + stats: Counter = Counter() + _backfill_annotation_timelines(db, allele_ids, batch_size=batch_size, run_at=run_at, stats=stats) + + print("\nRebuild completed:") + for key in ( + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + ): + print(f" {key}: {stats[key]}") + + +def verify_migration(db: Session, *, score_set_urn: Optional[str] = None) -> None: + """Report substrate coverage without writing.""" + print("\nVerifying backfill...") + + remaining = len(_variant_ids_to_migrate(db, score_set_urn)) + total_mapped_variant_variants = db.scalar(sa.select(sa.func.count(sa.distinct(MappedVariant.variant_id)))) + total_records = db.scalar(sa.select(sa.func.count(MappingRecord.id))) + live_records = db.scalar(sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.valid_to.is_(None))) + total_alleles = db.scalar(sa.select(sa.func.count(Allele.id))) + live_auth_links = db.scalar( + sa.select(sa.func.count(MappingRecordAllele.id)).where( + MappingRecordAllele.is_authoritative, MappingRecordAllele.valid_to.is_(None) + ) + ) + gnomad_links = db.scalar(sa.select(sa.func.count(GnomadAlleleLink.id))) + clinvar_links = db.scalar(sa.select(sa.func.count(ClinvarAlleleLink.id))) + vep_consequences = db.scalar(sa.select(sa.func.count(VepAlleleConsequence.id))) + migrated_events = db.scalar( + sa.select(sa.func.count(AnnotationEvent.id)).where(AnnotationEvent.reason == EventReason.MIGRATED.value) + ) + + print(f" Variants with mapped_variants: {total_mapped_variant_variants}") + print(f" Variants still needing backfill: {remaining}") + print(f" MappingRecords (total / live): {total_records} / {live_records}") + print(f" Alleles (deduped): {total_alleles}") + print(f" Live authoritative links: {live_auth_links}") + print(f" gnomAD / ClinVar / VEP annotations: {gnomad_links} / {clinvar_links} / {vep_consequences}") + print(f" MIGRATED annotation_event rows: {migrated_events}") + if remaining == 0: + print(" ✓ Every legacy variant has a MappingRecord.") + else: + print(f" ⚠ {remaining} variants remain (see logged anomalies, or re-run migrate).") + + +def rollback_migration(db: Session) -> None: + """Delete backfilled ``MappingRecord``s (cascading their allele links). + + Scoped to variants that also have ``mapped_variants`` — the exact discriminator + for backfilled records, since native new-model variants have no legacy rows. + Deduped ``Allele``s and annotation links are left in place: alleles are content + -addressed and harmless if orphaned (a re-run reuses them), and annotation + links re-existence-check. Delete those manually if a full teardown is needed. + + ``AnnotationEvent`` rows (reason ``MIGRATED``) are left in place too, on purpose: it is an + append-only audit log, and deleting them on rollback would erase the very audit trail it exists to + keep — a re-run after rollback just appends a fresh set (matching how ``MappingRecord`` itself gets + deleted and recreated), not a duplicate-avoidance concern the way the ValidTime domain tables are. + """ + print("Rolling back backfilled MappingRecords (scoped to variants with mapped_variants)...") + legacy_variant_ids = sa.select(sa.distinct(MappedVariant.variant_id)) + count = db.scalar( + sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.variant_id.in_(legacy_variant_ids)) + ) + # ON DELETE CASCADE on mapping_record_alleles.mapping_record_id removes the links. + db.execute(sa.delete(MappingRecord).where(MappingRecord.variant_id.in_(legacy_variant_ids))) + db.commit() + print(f"Deleted {count} MappingRecords (and their allele links via cascade).") + print("Note: deduped Alleles, annotation links, and MIGRATED annotation_event rows were left in place") + print("(re-runnable / harmless orphans; the event log is append-only by design).") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Backfill the Allele substrate from mapped_variants.") + parser.add_argument( + "command", nargs="?", default="migrate", choices=["migrate", "verify", "rollback", "rebuild-annotations"] + ) + parser.add_argument("--score-set", dest="score_set_urn", default=None, help="Limit to one score set URN.") + parser.add_argument("--batch-size", type=int, default=1000, help="Variants per commit (default 1000).") + parser.add_argument("--dry-run", action="store_true", help="Run without committing (rolls back each batch).") + return parser.parse_args() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + args = _parse_args() + + if args.command == "rollback": + print("WARNING: this deletes all backfilled MappingRecords and their allele links.") + if input("Are you sure you want to continue? (y/N): ").lower() == "y": + with SessionLocal() as session: + rollback_migration(session) + else: + print("Rollback cancelled.") + elif args.command == "verify": + with SessionLocal() as session: + verify_migration(session, score_set_urn=args.score_set_urn) + elif args.command == "rebuild-annotations": + with SessionLocal() as session: + rebuild_annotation_timelines(session, score_set_urn=args.score_set_urn, batch_size=args.batch_size) + verify_migration(session, score_set_urn=args.score_set_urn) + else: + with SessionLocal() as session: + do_migration( + session, + score_set_urn=args.score_set_urn, + batch_size=args.batch_size, + dry_run=args.dry_run, + ) + verify_migration(session, score_set_urn=args.score_set_urn) diff --git a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py index 4e793ffe5..41eec4665 100644 --- a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py +++ b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py @@ -76,7 +76,7 @@ from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import from mavedb.db.session import SessionLocal from mavedb.lib.variants import HGVS_G_REGEX, HGVS_P_REGEX, get_hgvs_from_post_mapped -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene import TargetGene @@ -102,18 +102,18 @@ _NUCLEOTIDE_ALPHABET = frozenset("ACGTUNRYSWKMBDHV-") -def _layer_from_hgvs(hgvs: str) -> Optional[AnnotationLayer]: - """Classify an HGVS string into an :class:`AnnotationLayer`.""" +def _layer_from_hgvs(hgvs: str) -> Optional[SequenceLevel]: + """Classify an HGVS string into an :class:`SequenceLevel`.""" if HGVS_G_REGEX.search(hgvs): - return AnnotationLayer.genomic + return SequenceLevel.genomic if HGVS_P_REGEX.search(hgvs): - return AnnotationLayer.protein + return SequenceLevel.protein if _HGVS_C_REGEX.search(hgvs): - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[AnnotationLayer]: +def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[SequenceLevel]: """ Infer layer from VRS sequence alphabet (protein vs nucleotide). Returns protein if any allele/member sequence contains a non-nucleotide letter. @@ -150,15 +150,15 @@ def extract_sequences(obj): for seq in extract_sequences(post_mapped): found_seq = True if any(letter.upper() not in _NUCLEOTIDE_ALPHABET for letter in seq if letter.isalpha()): - return AnnotationLayer.protein + return SequenceLevel.protein if found_seq: - return AnnotationLayer.genomic + return SequenceLevel.genomic return None -def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: +def _alignment_level_for(target: TargetGene) -> Optional[SequenceLevel]: """Recover the alignment layer for legacy mapped variants of ``target``. Tries ``post_mapped_metadata`` first, then ``pre_mapped_metadata`` (same @@ -177,16 +177,16 @@ def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: return None if has_genomic: - return AnnotationLayer.genomic + return SequenceLevel.genomic if has_protein: - return AnnotationLayer.protein + return SequenceLevel.protein if has_cdna: - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: +def _layer_from_sequence_type(target: TargetGene) -> Optional[SequenceLevel]: """Infer alignment layer from ``target_sequence.sequence_type`` and ``category``. Last-resort fallback for targets where all variant-level mappings failed @@ -201,16 +201,16 @@ def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: return None seq_type = target.target_sequence.sequence_type if seq_type == "protein": - return AnnotationLayer.protein + return SequenceLevel.protein if seq_type == "dna" and target.category is not None and target.category.value == "protein_coding": - return AnnotationLayer.genomic + return SequenceLevel.genomic return None def _populate_layer_by_target_id( db: Session, targets_by_score_set_id: dict[int, list[TargetGene]], - layer_by_target_id: dict[int, Optional[AnnotationLayer]], + layer_by_target_id: dict[int, Optional[SequenceLevel]], layer_via_hgvs_target_ids: set[int], layer_via_sequence_type_target_ids: set[int], ) -> None: @@ -344,7 +344,7 @@ def do_migration(db: Session) -> None: # Cache (target_gene_id, alignment_level, tool_version) -> persisted TargetGeneMapping # so we create one row per distinct combination and reuse it across every # mapped variant that maps into it. - cache: dict[tuple[int, AnnotationLayer, str], TargetGeneMapping] = {} + cache: dict[tuple[int, SequenceLevel, str], TargetGeneMapping] = {} total = db.scalar(sa.select(sa.func.count(MappedVariant.id))) or 0 print(f" {total} mapped variants to consider.") @@ -393,7 +393,7 @@ def do_migration(db: Session) -> None: # only need to determine it once per target -- and any single attributable mapped # variant for that target lets us attribute every sibling. The via_* sets track which # fallback was used so the diagnostic report shows attribution breadth. - layer_by_target_id: dict[int, Optional[AnnotationLayer]] = {} + layer_by_target_id: dict[int, Optional[SequenceLevel]] = {} layer_via_hgvs_target_ids: set[int] = set() layer_via_sequence_type_target_ids: set[int] = set() _populate_layer_by_target_id( @@ -420,7 +420,7 @@ def do_migration(db: Session) -> None: # Group ids to update by (tgm_id, level) so we can issue one bulk UPDATE # per group instead of a per-row ORM flush. - updates_by_group: dict[tuple[int, AnnotationLayer], list[int]] = defaultdict(list) + updates_by_group: dict[tuple[int, SequenceLevel], list[int]] = defaultdict(list) for row in chunk: last_id = row.id diff --git a/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py new file mode 100644 index 000000000..996badf85 --- /dev/null +++ b/alembic/versions/024370c4bca7_add_calibration_controls_table_and_phi_.py @@ -0,0 +1,113 @@ +"""add calibration controls, mondo terms, and phi/disease columns + +Revision ID: 024370c4bca7 +Revises: f6752571d14d +Create Date: 2026-09-15 16:01:06.013964 + +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "024370c4bca7" +down_revision = "f6752571d14d" +branch_labels = None +depends_on = None + +# The generic "disease or disorder" MONDO term seeded so the non-nullable calibration FK always resolves. +MONDO_SYSTEM = "https://purl.obolibrary.org/obo/mondo.owl" +MONDO_GENERIC_CODE = "MONDO:0000001" +MONDO_GENERIC_LABEL = "disease or disorder" + + +def upgrade(): + op.create_table( + "calibration_controls", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("calibration_id", sa.Integer(), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=False), + sa.Column( + "clinical_status", + sa.Enum("pathogenic", "benign", name="calibrationcontrolstatus", native_enum=False, length=32), + nullable=False, + ), + sa.Column("created_by_id", sa.Integer(), nullable=False), + sa.Column("modified_by_id", sa.Integer(), nullable=False), + sa.Column("creation_date", sa.Date(), nullable=False), + sa.Column("modification_date", sa.Date(), nullable=False), + sa.ForeignKeyConstraint(["calibration_id"], ["score_calibrations.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["variant_id"], ["variants.id"]), + sa.ForeignKeyConstraint(["created_by_id"], ["users.id"]), + sa.ForeignKeyConstraint(["modified_by_id"], ["users.id"]), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("calibration_id", "variant_id", name="uq_calibration_controls_calibration_id_variant_id"), + ) + op.create_index( + op.f("ix_calibration_controls_calibration_id"), "calibration_controls", ["calibration_id"], unique=False + ) + op.create_index(op.f("ix_calibration_controls_variant_id"), "calibration_controls", ["variant_id"], unique=False) + op.create_index( + op.f("ix_calibration_controls_created_by_id"), "calibration_controls", ["created_by_id"], unique=False + ) + op.create_index( + op.f("ix_calibration_controls_modified_by_id"), "calibration_controls", ["modified_by_id"], unique=False + ) + + # MONDO disease terms, with the generic root seeded as the default for calibrations naming no disease. + op.create_table( + "mondo_terms", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("code", sa.String(), nullable=False), + sa.Column("system", sa.String(), nullable=False), + sa.Column("system_version", sa.String(), nullable=True), + sa.Column("label", sa.String(), nullable=False), + sa.Column("creation_date", sa.Date(), nullable=False), + sa.Column("modification_date", sa.Date(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("system", "code", name="uq_mondo_terms_system_code"), + ) + op.execute( + sa.text( + "INSERT INTO mondo_terms (code, system, label, creation_date, modification_date) " + "VALUES (:code, :system, :label, CURRENT_DATE, CURRENT_DATE)" + ).bindparams(code=MONDO_GENERIC_CODE, system=MONDO_SYSTEM, label=MONDO_GENERIC_LABEL) + ) + + # Disease FK: add nullable, backfill existing calibrations to the generic term, then enforce NOT NULL. + op.add_column("score_calibrations", sa.Column("disease_term_id", sa.Integer(), nullable=True)) + op.execute( + sa.text( + "UPDATE score_calibrations SET disease_term_id = " + "(SELECT id FROM mondo_terms WHERE system = :system AND code = :code)" + ).bindparams(system=MONDO_SYSTEM, code=MONDO_GENERIC_CODE) + ) + op.alter_column("score_calibrations", "disease_term_id", nullable=False) + op.create_index( + op.f("ix_score_calibrations_disease_term_id"), "score_calibrations", ["disease_term_id"], unique=False + ) + op.create_foreign_key( + "fk_score_calibrations_disease_term_id_mondo_terms", + "score_calibrations", + "mondo_terms", + ["disease_term_id"], + ["id"], + ) + + op.add_column("score_calibrations", sa.Column("controls_not_phi", sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column("score_calibrations", "controls_not_phi") + + op.drop_constraint("fk_score_calibrations_disease_term_id_mondo_terms", "score_calibrations", type_="foreignkey") + op.drop_index(op.f("ix_score_calibrations_disease_term_id"), table_name="score_calibrations") + op.drop_column("score_calibrations", "disease_term_id") + op.drop_table("mondo_terms") + + op.drop_index(op.f("ix_calibration_controls_modified_by_id"), table_name="calibration_controls") + op.drop_index(op.f("ix_calibration_controls_created_by_id"), table_name="calibration_controls") + op.drop_index(op.f("ix_calibration_controls_variant_id"), table_name="calibration_controls") + op.drop_index(op.f("ix_calibration_controls_calibration_id"), table_name="calibration_controls") + op.drop_table("calibration_controls") diff --git a/alembic/versions/15c40c367731_index_functional_classification_variant_.py b/alembic/versions/15c40c367731_index_functional_classification_variant_.py new file mode 100644 index 000000000..5140e3b7a --- /dev/null +++ b/alembic/versions/15c40c367731_index_functional_classification_variant_.py @@ -0,0 +1,39 @@ +"""index functional classification variant fk + +variant_id is only the trailing column of score_calibration_functional_classification_variants' +composite primary key, so Postgres has no index it can use to look up rows by variant_id alone. +Every Variant delete (including the per-variant cascade when a score set is deleted) forces a +sequential scan of this table to enforce that FK, which dominates the delete for any score set +with a nontrivial number of variants. See mavedb-api#677 ("unpublished score set deletion often +fails"). + +Building it takes a lock that blocks writes to the table, so in a populated database build it by hand +first, which makes this migration a no-op: + + CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_score_calib_func_classification_variants_variant_id + ON score_calibration_functional_classification_variants (variant_id); + +Revision ID: 15c40c367731 +Revises: c4b18d0f7a92 +Create Date: 2026-09-21 11:19:39.396343 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "15c40c367731" +down_revision = "c4b18d0f7a92" +branch_labels = None +depends_on = None + +INDEX_NAME = "ix_score_calib_func_classification_variants_variant_id" +TABLE_NAME = "score_calibration_functional_classification_variants" + + +def upgrade(): + op.execute(f"CREATE INDEX IF NOT EXISTS {INDEX_NAME} ON {TABLE_NAME} (variant_id)") + + +def downgrade(): + op.execute(f"DROP INDEX IF EXISTS {INDEX_NAME}") diff --git a/alembic/versions/393764b9e833_cascade_annotation_event_variant_delete.py b/alembic/versions/393764b9e833_cascade_annotation_event_variant_delete.py new file mode 100644 index 000000000..95450a795 --- /dev/null +++ b/alembic/versions/393764b9e833_cascade_annotation_event_variant_delete.py @@ -0,0 +1,43 @@ +"""cascade annotation events when their variant is deleted + +Revision ID: 393764b9e833 +Revises: 45e8ec1db15e +Create Date: 2026-09-29 + +The mapping job records a variant-subject ``annotation_event`` for every variant it maps. With the foreign key +on ``RESTRICT``, deleting a mapped score set or re-uploading its variants failed on that key. Variant-subject +events now go with their variant, as ``variant_annotation_status`` rows do. Allele-subject events keep +``RESTRICT``: alleles are shared across score sets and are never deleted with one. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "393764b9e833" +down_revision = "45e8ec1db15e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint("annotation_event_variant_id_fkey", "annotation_event", type_="foreignkey") + op.create_foreign_key( + "annotation_event_variant_id_fkey", + "annotation_event", + "variants", + ["variant_id"], + ["id"], + ondelete="CASCADE", + ) + + +def downgrade() -> None: + op.drop_constraint("annotation_event_variant_id_fkey", "annotation_event", type_="foreignkey") + op.create_foreign_key( + "annotation_event_variant_id_fkey", + "annotation_event", + "variants", + ["variant_id"], + ["id"], + ondelete="RESTRICT", + ) diff --git a/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py b/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py new file mode 100644 index 000000000..92d88dd81 --- /dev/null +++ b/alembic/versions/45e8ec1db15e_carry_score_set_id_on_mapping_tables.py @@ -0,0 +1,94 @@ +"""carry score_set_id on mapping_records and mapping_record_alleles + +Revision ID: 45e8ec1db15e +Revises: d1c9b7a3e2f4 +Create Date: 2026-09-28 + +Adds a ``score_set_id`` column to ``mapping_records`` and ``mapping_record_alleles`` so an RLS policy can +check a row's score set directly (#833). Both tables otherwise reach their score set only through +``variants``, and delegating through a table that size costs ~700 ms per lookup. + +Each column sits under a composite foreign key to its parent's id and score set, so it can't disagree +with the variant it belongs to. The composite keys replace the single-column ``variant_id`` and +``mapping_record_id`` foreign keys. Both cascade on delete, so deleting a variant (on re-upload or score set +deletion) removes its mapping records and their allele links. + +**Before deploying to production**, build the ``variants`` index by hand, since a migration can't build it +concurrently (#877). The ``IF NOT EXISTS`` below then does nothing there, and builds the index on an empty +or small database:: + + CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS uq_variants_id_scoreset ON variants (id, scoreset_id); + +The backfills are sized for staging, where the mapping tables already hold data. In production the tables +are empty when this runs. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "45e8ec1db15e" +down_revision = "d1c9b7a3e2f4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE UNIQUE INDEX IF NOT EXISTS uq_variants_id_scoreset ON variants (id, scoreset_id)") + + op.execute("ALTER TABLE mapping_records ADD COLUMN score_set_id integer") + op.execute("UPDATE mapping_records mr SET score_set_id = v.scoreset_id FROM variants v WHERE v.id = mr.variant_id") + op.execute("ALTER TABLE mapping_records ALTER COLUMN score_set_id SET NOT NULL") + op.execute("CREATE UNIQUE INDEX uq_mapping_records_id_score_set ON mapping_records (id, score_set_id)") + op.drop_constraint("fk_mapping_records_variant_id", "mapping_records", type_="foreignkey") + op.create_foreign_key( + "fk_mapping_records_variant_score_set", + "mapping_records", + "variants", + ["variant_id", "score_set_id"], + ["id", "scoreset_id"], + ondelete="CASCADE", + ) + + op.execute("ALTER TABLE mapping_record_alleles ADD COLUMN score_set_id integer") + op.execute( + "UPDATE mapping_record_alleles mra SET score_set_id = mr.score_set_id " + "FROM mapping_records mr WHERE mr.id = mra.mapping_record_id" + ) + op.execute("ALTER TABLE mapping_record_alleles ALTER COLUMN score_set_id SET NOT NULL") + op.drop_constraint("fk_mapping_record_alleles_mapping_record_id", "mapping_record_alleles", type_="foreignkey") + op.create_foreign_key( + "fk_mapping_record_alleles_mapping_record_score_set", + "mapping_record_alleles", + "mapping_records", + ["mapping_record_id", "score_set_id"], + ["id", "score_set_id"], + ondelete="CASCADE", + ) + + +def downgrade() -> None: + op.drop_constraint( + "fk_mapping_record_alleles_mapping_record_score_set", "mapping_record_alleles", type_="foreignkey" + ) + op.create_foreign_key( + "fk_mapping_record_alleles_mapping_record_id", + "mapping_record_alleles", + "mapping_records", + ["mapping_record_id"], + ["id"], + ondelete="CASCADE", + ) + op.drop_column("mapping_record_alleles", "score_set_id") + + op.drop_constraint("fk_mapping_records_variant_score_set", "mapping_records", type_="foreignkey") + op.create_foreign_key( + "fk_mapping_records_variant_id", + "mapping_records", + "variants", + ["variant_id"], + ["id"], + ) + op.drop_index("uq_mapping_records_id_score_set", table_name="mapping_records") + op.drop_column("mapping_records", "score_set_id") + + op.drop_index("uq_variants_id_scoreset", table_name="variants") diff --git a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py index 479c51a33..1ce888933 100644 --- a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py +++ b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py @@ -8,8 +8,6 @@ from alembic import op -from mavedb.models.published_variant import signature - # revision identifiers, used by Alembic. revision = "4726e4dddde8" @@ -17,6 +15,11 @@ branch_labels = None depends_on = None +# Frozen literal, not imported from the model: this migration replaces the historical +# mapped_variants-based MV indexes at its point in history. A later migration rebuilds the MV onto +# the mapping_records substrate and re-indexes it, so the column names below stay the legacy ones. +signature = "published_variants_materialized_view" + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### diff --git a/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py b/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py new file mode 100644 index 000000000..6c1fc1d99 --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py @@ -0,0 +1,151 @@ +"""Add mapping_records, alleles, and mapping_record_alleles tables + +Revision ID: a1b2c3d4e5f6 +Revises: a7f3c2e9b104 +Create Date: 2026-05-29 + +New parallel tables for the Better Reverse Translation epic (#746). +The existing mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1b2c3d4e5f6" +down_revision = "a7f3c2e9b104" +branch_labels = None +depends_on = None + +VALID_ASSAY_LEVELS = "('genomic', 'cdna', 'protein')" +VALID_ALIGNMENT_LEVELS = "('protein', 'cdna', 'genomic')" + + +def upgrade() -> None: + op.create_table( + "alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=False), + sa.Column("level", sa.String(length=16), nullable=False), + sa.Column("transcript", sa.String(), nullable=False), + sa.Column("hgvs_g", sa.String(), nullable=True), + sa.Column("hgvs_c", sa.String(), nullable=True), + sa.Column("hgvs_p", sa.String(), nullable=True), + sa.Column("clingen_allele_id", sa.String(), nullable=True), + sa.Column("post_mapped", postgresql.JSONB(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + ) + op.create_index("ix_alleles_vrs_digest", "alleles", ["vrs_digest"]) + op.create_index("ix_alleles_level", "alleles", ["level"]) + op.create_index("ix_alleles_clingen_allele_id", "alleles", ["clingen_allele_id"]) + + op.create_table( + "mapping_records", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=True), + sa.Column("pre_mapped", postgresql.JSONB(), nullable=True), + sa.Column("assay_level", sa.String(length=16), nullable=False), + sa.Column("hgvs_assay_level", sa.String(), nullable=True), + sa.Column("mapping_api_version", sa.String(), nullable=False), + sa.Column("mapped_date", sa.Date(), nullable=False), + sa.Column("vrs_version", sa.String(), nullable=True), + sa.Column("current", sa.Boolean(), nullable=False), + sa.Column("alignment_level", sa.String(length=16), nullable=True), + sa.Column("at_mismatched_locus", sa.Boolean(), nullable=True), + sa.Column("near_gap", sa.Boolean(), nullable=True), + sa.Column("target_gene_mapping_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.ForeignKeyConstraint( + ["variant_id"], + ["variants.id"], + name="fk_mapping_records_variant_id", + ), + sa.ForeignKeyConstraint( + ["target_gene_mapping_id"], + ["target_gene_mappings.id"], + name="fk_mapping_records_target_gene_mapping_id", + ), + sa.PrimaryKeyConstraint("id"), + sa.CheckConstraint( + f"assay_level IN {VALID_ASSAY_LEVELS}", + name="ck_mapping_records_assay_level_valid", + ), + ) + op.create_index("ix_mapping_records_variant_id", "mapping_records", ["variant_id"]) + op.create_index("ix_mapping_records_vrs_digest", "mapping_records", ["vrs_digest"]) + op.create_index( + "ix_mapping_records_target_gene_mapping_id", + "mapping_records", + ["target_gene_mapping_id"], + ) + + op.create_table( + "mapping_record_alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("mapping_record_id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column( + "is_authoritative", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + sa.ForeignKeyConstraint( + ["mapping_record_id"], + ["mapping_records.id"], + name="fk_mapping_record_alleles_mapping_record_id", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_mapping_record_alleles_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_alleles", + ["mapping_record_id"], + ) + op.create_index( + "ix_mapping_record_alleles_allele_id", + "mapping_record_alleles", + ["allele_id"], + ) + + +def downgrade() -> None: + op.drop_index("ix_mapping_record_alleles_allele_id", table_name="mapping_record_alleles") + op.drop_index("ix_mapping_record_alleles_mapping_record_id", table_name="mapping_record_alleles") + op.drop_table("mapping_record_alleles") + + op.drop_index("ix_mapping_records_target_gene_mapping_id", table_name="mapping_records") + op.drop_index("ix_mapping_records_vrs_digest", table_name="mapping_records") + op.drop_index("ix_mapping_records_variant_id", table_name="mapping_records") + op.drop_table("mapping_records") + + op.drop_index("ix_alleles_clingen_allele_id", table_name="alleles") + op.drop_index("ix_alleles_level", table_name="alleles") + op.drop_index("ix_alleles_vrs_digest", table_name="alleles") + op.drop_table("alleles") diff --git a/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py new file mode 100644 index 000000000..c8f702d72 --- /dev/null +++ b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py @@ -0,0 +1,91 @@ +"""rewrite v_variant_annotations onto the mapping_records/alleles substrate + +Revision ID: a1d4f7c2e9b0 +Revises: e7b2c9a1f4d3 +Create Date: 2026-07-18 + +Drops the legacy ``mapped_variants``-based ``v_variant_annotations`` view and recreates it over the +``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate (part of the MappedVariant +read-path teardown). Column changes vs. the legacy shape: + * ``mapped_variant_id`` -> ``mapping_record_id`` (the mapping identity is now the mapping record). + * ``mapping_error`` dropped — there is no per-mapping-record error column; per-subject annotation + dispositions live in ``v_current_annotation_events`` and score-set-level mapping errors on + ``scoresets.mapping_errors``. + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``variant_annotation_view.definition`` would +couple this fixed revision to mutable code, so the next model edit (the following view rewrite) would +make a replay of *this* revision build that later shape — against tables that may not exist yet at this +point in history — and fail. ``_UPGRADE_DEFINITION`` below was compiled once from the model at +authoring time (``CreateView(signature, definition).compile(postgresql.dialect())``) and must not be +edited to track the model; the next shape change gets its own migration. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1d4f7c2e9b0" +down_revision = "e7b2c9a1f4d3" +branch_labels = None +depends_on = None + +SIGNATURE = "v_variant_annotations" + +# New (mapping_records/alleles) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. Allele-derived columns are correlated scalar +# subqueries so the grain stays one row per (variant, annotation_type), matching the legacy view. +_UPGRADE_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapping_records.id AS mapping_record_id, (SELECT alleles.clingen_allele_id +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS clingen_allele_id, mapping_records.hgvs_assay_level, (SELECT alleles.hgvs_g +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'genomic' + LIMIT 1) AS hgvs_g, (SELECT alleles.hgvs_c +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'cdna' + LIMIT 1) AS hgvs_c, (SELECT alleles.hgvs_p +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'protein' + LIMIT 1) AS hgvs_p, (SELECT vep_allele_consequences.functional_consequence +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_functional_consequence, (SELECT vep_allele_consequences.access_date +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_access_date, mapping_records.mapped_date, mapping_records.mapping_api_version, mapping_records.vrs_version, variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapping_records ON mapping_records.variant_id = variants.id AND mapping_records.valid_to IS NULL + LEFT OUTER JOIN variant_annotation_status ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + +# Legacy (mapped_variants-based) definition, inlined so the downgrade can restore the prior shape. +# ``mapped_variants`` still exists at this revision (dropped in a later teardown migration), so this +# remains runnable on downgrade. +_LEGACY_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + + +def upgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_UPGRADE_DEFINITION}") + + +def downgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_LEGACY_DEFINITION}") diff --git a/alembic/versions/a3f9c2e1b7d6_add_vep_consequence_provenance.py b/alembic/versions/a3f9c2e1b7d6_add_vep_consequence_provenance.py new file mode 100644 index 000000000..ff1790bf3 --- /dev/null +++ b/alembic/versions/a3f9c2e1b7d6_add_vep_consequence_provenance.py @@ -0,0 +1,73 @@ +"""add resolution provenance to vep_allele_consequences + +Revision ID: a3f9c2e1b7d6 +Revises: b2e5a8d3f0c1 +Create Date: 2026-08-26 + +Adds four nullable columns to ``vep_allele_consequences`` so a stored consequence records *how* it was +resolved, not just the headline term (#772): + +- ``consequence_terms`` (JSONB) — every term from the matched transcript entry, severity-ordered. +- ``consequence_source`` (varchar check-constraint enum) — ``transcript`` | ``most_severe`` | + ``reference_identical``. +- ``matched_transcript`` (varchar) — the versioned transcript VEP used; NULL unless the source is + ``transcript``. +- ``resolver_version`` (varchar) — ``variant_annotation.lib.vep.RESOLVER_VERSION``, the version of our + resolution *rule* (severity ranking, transcript-matching, Recoder combination). Pairs with + ``source_version`` (the Ensembl release) as the second version axis the current-release skip keys on, + so a rule fix shipped in the shared kernel re-queries every allele instead of looking current forever. + +All nullable, no backfill: rows written before this migration keep NULL provenance / resolver_version. +A NULL ``resolver_version`` never matches the current one, so a legacy row is re-queried on the next VEP +run and then filled in place (the linker supersedes on a value change, and a NULL→populated column for +an unchanged headline term advances in place rather than churning history). +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a3f9c2e1b7d6" +down_revision = "b2e5a8d3f0c1" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "vep_allele_consequences", + sa.Column("consequence_terms", postgresql.JSONB(none_as_null=True), nullable=True), + ) + op.add_column( + "vep_allele_consequences", + sa.Column( + "consequence_source", + sa.Enum( + "transcript", + "most_severe", + "reference_identical", + name="vepconsequencesource", + native_enum=False, + create_constraint=True, + length=32, + ), + nullable=True, + ), + ) + op.add_column( + "vep_allele_consequences", + sa.Column("matched_transcript", sa.String(), nullable=True), + ) + op.add_column( + "vep_allele_consequences", + sa.Column("resolver_version", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("vep_allele_consequences", "resolver_version") + op.drop_column("vep_allele_consequences", "matched_transcript") + op.drop_column("vep_allele_consequences", "consequence_source") + op.drop_column("vep_allele_consequences", "consequence_terms") diff --git a/alembic/versions/a7c3e5f9b1d2_unique_allele_hgvs_expression.py b/alembic/versions/a7c3e5f9b1d2_unique_allele_hgvs_expression.py new file mode 100644 index 000000000..1ead4d392 --- /dev/null +++ b/alembic/versions/a7c3e5f9b1d2_unique_allele_hgvs_expression.py @@ -0,0 +1,64 @@ +"""enforce one allele per HGVS expression + +Revision ID: a7c3e5f9b1d2 +Revises: 024370c4bca7 +Create Date: 2026-09-30 + +Adds a unique index on the allele's HGVS expression (whichever of hgvs_g / hgvs_c / hgvs_p is set; the +model guarantees exactly one). An allele digest covers the refget of the sequence the allele sits on, +and alleles deduplicate on that digest. The same expression under two digests therefore means two +writers built the allele on different sequences (this happened with NM_007294.3 and NP_001346945.1 when +the mapper and reverse translation resolved one accession to different sequences), and the copies never +merge: reverse translation's fold-in misses the measured allele and the convergent / projection +labelling breaks. Nothing else detects it, because both digests are internally consistent. This index +moves the failure to write time, in get_or_create_allele, which raises AlleleIdentityConflictError with +both digests, both refgets and how to debug it. + +Built CONCURRENTLY: mapping and reverse translation write to alleles, and a plain CREATE UNIQUE INDEX +would block them. A concurrent build that fails leaves an INVALID index behind, which must be dropped +before retrying: + + DROP INDEX CONCURRENTLY IF EXISTS uq_alleles_hgvs; + +Assumes no two alleles share an HGVS expression. Check first: + + SELECT coalesce(hgvs_g, hgvs_c, hgvs_p) AS hgvs, count(*), array_agg(id ORDER BY id) + FROM alleles + GROUP BY 1 HAVING count(*) > 1; + +If that returns rows, remove the duplicates before applying this migration. In each group keep the +allele whose refget is the one SeqRepo holds for its accession. Delete the other allele and the rows +that reference it (mapping_record_alleles, gnomad_allele_links, vep_allele_consequences, +clinvar_allele_links, annotation_event; all ON DELETE RESTRICT), then remap the affected score sets. +Delete before remapping: get_or_create_allele refuses to create an allele while another holds its HGVS. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7c3e5f9b1d2" +down_revision = "024370c4bca7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + op.create_index( + "uq_alleles_hgvs", + "alleles", + [sa.text("(coalesce(hgvs_g, hgvs_c, hgvs_p))")], + unique=True, + postgresql_concurrently=True, + ) + op.execute( + "COMMENT ON INDEX uq_alleles_hgvs IS " + "'One allele per HGVS expression. A violation means two writers disagree about the sequence an " + "allele sits on (see mavedb.lib.variant_translations.AlleleIdentityConflictError).'" + ) + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + op.drop_index("uq_alleles_hgvs", table_name="alleles", postgresql_concurrently=True) diff --git a/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py new file mode 100644 index 000000000..8b655f625 --- /dev/null +++ b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py @@ -0,0 +1,65 @@ +"""add vep_allele_consequences table + +Revision ID: a7c4e9d2f1b8 +Revises: e5f7a9c1b3d4 +Create Date: 2026-06-22 + +New valid-time table holding a deduplicated allele's VEP functional consequence, replacing the frozen +vep_functional_consequence/vep_access_date columns on mapped_variants for new-model writes (Step 2 of +the annotation infrastructure migration, docs/design/annotation-infrastructure-migration.md). A row is +live while valid_to is NULL; the partial unique index enforces a single live consequence per allele +(VEP's most-severe consequence is one current value, so a change supersedes rather than accumulates). +functional_consequence is nullable (reserved for a future negative cache). source_version is the +Ensembl release the consequence was resolved under (coordinated software + transcript set + vocabulary), +which version-keys the refresh skip like gnomAD's db_version; access_date is retained as a "last +confirmed" audit stamp. The VEP columns on mapped_variants are left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7c4e9d2f1b8" +down_revision = "e5f7a9c1b3d4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "vep_allele_consequences", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("functional_consequence", sa.String(), nullable=True), + sa.Column("source_version", sa.String(), nullable=False), + sa.Column("access_date", sa.Date(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_vep_allele_consequences_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_vep_allele_consequences_allele_id", + "vep_allele_consequences", + ["allele_id"], + ) + # One live consequence per allele: VEP's most-severe consequence is a single current value, so a + # changed result supersedes the prior row rather than accumulating one live row per access. + op.create_index( + "uq_vep_allele_consequences_live", + "vep_allele_consequences", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_vep_allele_consequences_live", table_name="vep_allele_consequences") + op.drop_index("ix_vep_allele_consequences_allele_id", table_name="vep_allele_consequences") + op.drop_table("vep_allele_consequences") diff --git a/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py new file mode 100644 index 000000000..cf8d77e35 --- /dev/null +++ b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py @@ -0,0 +1,79 @@ +"""add annotation_event log + +Revision ID: a7e1c4f9b3d2 +Revises: d4e5f6a7b8c9 +Create Date: 2026-06-25 16:00:00.000000 + +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7e1c4f9b3d2" +down_revision = "d4e5f6a7b8c9" +branch_labels = None +depends_on = None + +_VARIANT_SUBJECT_TYPES = "'vrs_mapping', 'cross_level_translation', 'variant_translation', 'ldh_submission'" +_ALLELE_SUBJECT_TYPES = ( + "'clingen_allele_id', 'gnomad_allele_frequency', 'vep_functional_consequence', 'clinvar_control', 'mapped_hgvs'" +) + + +def upgrade(): + op.create_table( + "annotation_event", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("annotation_type", sa.String(length=50), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=True), + sa.Column("allele_id", sa.Integer(), nullable=True), + sa.Column("disposition", sa.String(length=50), nullable=False), + sa.Column("reason", sa.String(length=50), nullable=False), + sa.Column("source_version", sa.String(length=50), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("job_run_id", sa.Integer(), nullable=True), + sa.Column("score_set_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.CheckConstraint( + f"(annotation_type IN ({_VARIANT_SUBJECT_TYPES}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_ALLELE_SUBJECT_TYPES}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + sa.ForeignKeyConstraint(["variant_id"], ["variants.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["allele_id"], ["alleles.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["job_run_id"], ["job_runs.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["score_set_id"], ["scoresets.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_annotation_event_allele_type_id", + "annotation_event", + ["allele_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_variant_type_id", + "annotation_event", + ["variant_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_allele_type_version", + "annotation_event", + ["allele_id", "annotation_type", "source_version"], + unique=False, + ) + op.create_index("ix_annotation_event_job_run_id", "annotation_event", ["job_run_id"], unique=False) + + +def downgrade(): + op.drop_index("ix_annotation_event_job_run_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_version", table_name="annotation_event") + op.drop_index("ix_annotation_event_variant_type_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_id", table_name="annotation_event") + op.drop_table("annotation_event") diff --git a/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py new file mode 100644 index 000000000..d04ceab9b --- /dev/null +++ b/alembic/versions/a7f3c2e9b104_add_progress_updated_at_to_job_runs.py @@ -0,0 +1,44 @@ +"""Add progress_updated_at heartbeat column to job_runs + +Revision ID: a7f3c2e9b104 +Revises: 4dbf24ed1857 +Create Date: 2026-07-09 00:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "a7f3c2e9b104" +down_revision = "4dbf24ed1857" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "job_runs", + sa.Column( + "progress_updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + ) + op.create_index( + "ix_job_runs_status_progress_updated", + "job_runs", + ["status", "progress_updated_at"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_job_runs_status_progress_updated", table_name="job_runs") + op.drop_column("job_runs", "progress_updated_at") + # ### end Alembic commands ### diff --git a/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py new file mode 100644 index 000000000..19cca0789 --- /dev/null +++ b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py @@ -0,0 +1,39 @@ +"""rename clinical_controls to clinvar_controls + +Revision ID: b2c3d4e5f6a7 +Revises: a7c4e9d2f1b8 +Create Date: 2026-06-22 + +Renames the clinical_controls entity table to clinvar_controls, and renames the unique +constraint to match. The frozen association table (mapped_variants_clinical_controls) +and its FK to the renamed table are left structurally intact — PostgreSQL updates the FK +target automatically on table rename. The Python model is renamed ClinicalControl → +ClinvarControl in the same changeset (no data migration). +""" + +from alembic import op + +revision = "b2c3d4e5f6a7" +down_revision = "a7c4e9d2f1b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.rename_table("clinical_controls", "clinvar_controls") + # PostgreSQL does not auto-rename constraints on table rename; rename explicitly so the + # on_conflict_do_update(constraint=...) in the job references the correct name. + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinical_controls_db_name_identifier_version " + "TO uq_clinvar_controls_db_name_identifier_version" + ) + + +def downgrade() -> None: + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinvar_controls_db_name_identifier_version " + "TO uq_clinical_controls_db_name_identifier_version" + ) + op.rename_table("clinvar_controls", "clinical_controls") diff --git a/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py new file mode 100644 index 000000000..2f9dad721 --- /dev/null +++ b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py @@ -0,0 +1,86 @@ +"""rebuild published_variants MV onto the mapping_records substrate + +Revision ID: b2e5a8d3f0c1 +Revises: a1d4f7c2e9b0 +Create Date: 2026-07-18 + +Rebuilds the ``published_variants_materialized_view`` off the legacy ``mapped_variants`` join onto the +``mapping_records`` substrate (part of the MappedVariant read-path teardown). The MV keeps its grain +(one row per published variant per mapping-record version, unmapped variants retained with a NULL +mapping id) and its published filter. Column changes: + * ``mapped_variant_id`` -> ``mapping_record_id`` + * ``current_mapped_variant`` -> ``current_mapping_record`` (``valid_to IS NULL``) + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``published_variant.definition`` would couple +this fixed revision to mutable code, so a later model edit would make a replay of *this* revision build +that later shape (possibly against tables that no longer exist by then) and fail. ``_UPGRADE_DEFINITION`` +below was compiled once from the model at authoring time and must not be edited to track the model. +""" + +from alembic_utils.pg_materialized_view import PGMaterializedView + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b2e5a8d3f0c1" +down_revision = "a1d4f7c2e9b0" +branch_labels = None +depends_on = None + +SIGNATURE = "published_variants_materialized_view" + +# New (mapping_records-based) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. The LEFT OUTER JOIN to mapping_records carries no +# liveness filter on purpose — every record version is retained and current-ness is exposed via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). +_UPGRADE_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapping_records.id AS mapping_record_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapping_records.valid_to IS NULL AS current_mapping_record +FROM variants LEFT OUTER JOIN mapping_records ON variants.id = mapping_records.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + +# Legacy (mapped_variants-based) definition, inlined for the downgrade path. ``mapped_variants`` still +# exists at this revision (dropped in a later teardown migration), so downgrade remains runnable. +_LEGACY_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + + +def upgrade(): + # Dropping the MV cascades to its owned indexes (the legacy idx_..._ids on mapped_variant_id). + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapping_record_id", "variant_id", "score_set_id"], + unique=True, + ) + + +def downgrade(): + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapped_variant_id", "variant_id", "score_set_id"], + unique=True, + ) diff --git a/alembic/versions/b4d8f2a6c1e3_score_set_score_distribution.py b/alembic/versions/b4d8f2a6c1e3_score_set_score_distribution.py new file mode 100644 index 000000000..364018af9 --- /dev/null +++ b/alembic/versions/b4d8f2a6c1e3_score_set_score_distribution.py @@ -0,0 +1,30 @@ +"""store a binned score distribution on each score set + +Revision ID: b4d8f2a6c1e3 +Revises: a7c3e5f9b1d2 +Create Date: 2026-10-08 + +Adds a nullable score_distribution JSONB column, written by the variant creation job (see +mavedb.lib.score_distribution). Adding a nullable column without a default is a catalog-only change and +does not rewrite the table. Existing score sets stay null until the backfill runs: + + python -m mavedb.scripts.backfill_score_distributions --commit +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "b4d8f2a6c1e3" +down_revision = "a7c3e5f9b1d2" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("scoresets", sa.Column("score_distribution", postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + + +def downgrade(): + op.drop_column("scoresets", "score_distribution") diff --git a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py index 7ce51c88d..3b3855e7a 100644 --- a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py +++ b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py @@ -2,13 +2,16 @@ Revision ID: b85bc7b1bec7 Revises: c404b6719110 Create Date: 2025-03-14 01:53:19.898198 + +The MV SQL is inlined as a frozen literal rather than imported from the model. At this revision the +MV joins the (now legacy) ``mapped_variants`` table; a later migration rebuilds it onto the +``mapping_records`` substrate. Importing the live model definition here would make a fresh replay +build *that* later shape at this early revision — before the substrate tables exist — and fail. +Freezing the historical SQL keeps replay green; the rebuild happens in its own migration. """ from alembic import op from alembic_utils.pg_materialized_view import PGMaterializedView -from sqlalchemy.dialects import postgresql - -from mavedb.models.published_variant import signature, definition # revision identifiers, used by Alembic. @@ -17,59 +20,71 @@ branch_labels = None depends_on = None +SIGNATURE = "published_variants_materialized_view" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + def upgrade(): op.create_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) op.create_index( - f"idx_{signature}_variant_id", - signature, + f"idx_{SIGNATURE}_variant_id", + SIGNATURE, ["variant_id"], unique=False, ) op.create_index( - f"idx_{signature}_variant_urn", - signature, + f"idx_{SIGNATURE}_variant_urn", + SIGNATURE, ["variant_urn"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_id", - signature, + f"idx_{SIGNATURE}_score_set_id", + SIGNATURE, ["score_set_id"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_urn", - signature, + f"idx_{SIGNATURE}_score_set_urn", + SIGNATURE, ["score_set_urn"], unique=False, ) op.create_index( - f"idx_{signature}_mapped_variant_id", - signature, + f"idx_{SIGNATURE}_mapped_variant_id", + SIGNATURE, ["mapped_variant_id"], unique=True, ) def downgrade(): - op.drop_index(f"idx_{signature}_variant_id", signature) - op.drop_index(f"idx_{signature}_variant_urn", signature) - op.drop_index(f"idx_{signature}_mapped_variant_id", signature) - op.drop_index(f"idx_{signature}_score_set_id", signature) - op.drop_index(f"idx_{signature}_score_set_urn", signature) + op.drop_index(f"idx_{SIGNATURE}_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_variant_urn", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_mapped_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_urn", SIGNATURE) op.drop_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) diff --git a/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py new file mode 100644 index 000000000..2705d9a3c --- /dev/null +++ b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py @@ -0,0 +1,32 @@ +"""drop alleles.transcript column + +Revision ID: b8e1f0a2c4d7 +Revises: f4d2a9c1b7e3 +Create Date: 2026-06-05 + +The `transcript` column duplicated data already present in the HGVS columns — it was +always extract_accession(hgvs_g/hgvs_c/hgvs_p). It is now a derived hybrid_property on +the Allele model (split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)), so the stored +column is removed to keep a single source of truth and avoid drift. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b8e1f0a2c4d7" +down_revision = "f4d2a9c1b7e3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_column("alleles", "transcript") + + +def downgrade() -> None: + # Re-add the column and backfill it from the HGVS columns (the same derivation the + # hybrid_property uses) so the restored NOT NULL column is consistent. + op.add_column("alleles", sa.Column("transcript", sa.String(), nullable=True)) + op.execute("UPDATE alleles SET transcript = split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)") + op.alter_column("alleles", "transcript", nullable=False) diff --git a/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py new file mode 100644 index 000000000..73843866c --- /dev/null +++ b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py @@ -0,0 +1,30 @@ +"""add v_current_annotation_events view + +Revision ID: b8f2c5a1d3e4 +Revises: a7e1c4f9b3d2 +Create Date: 2026-06-26 + +Creates the v_current_annotation_events view: the latest AnnotationEvent per (subject, annotation_type), +with ClinVar partitioned additionally by source_version (multi-live, one current row per release). +Replaces the per-variant v_variant_annotations as the current-state projection over the new +allele-model annotation log. Intended for operator queries, BI, and the annotation CLI scripts. +""" + +from alembic import op + +from mavedb.db.view import CreateView, DropView +from mavedb.models.annotation_event_view import definition, signature + +# revision identifiers, used by Alembic. +revision = "b8f2c5a1d3e4" +down_revision = "a7e1c4f9b3d2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(CreateView(signature, definition, materialized=False)) + + +def downgrade() -> None: + op.execute(DropView(signature, materialized=False)) diff --git a/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py new file mode 100644 index 000000000..4a20f008f --- /dev/null +++ b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py @@ -0,0 +1,73 @@ +"""add clinvar_allele_links table + +Revision ID: c3d4e5f6a7b8 +Revises: b2c3d4e5f6a7 +Create Date: 2026-06-22 + +New valid-time link table connecting deduplicated alleles to ClinvarControl rows, replacing +the frozen mapped_variants_clinical_controls association for new-model writes. + +ClinVar's link shape is deliberately multi-live: the partial unique index is +(allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live +link per ClinVar release rather than superseding as in gnomAD/VEP. Each ClinVar release is a +distinct ClinvarControl row, so different releases stack as independent live links. A link is +retired (valid_to closed) only if ClinVar later removes the variant from a release, which +would surface as a re-run finding no data for that release and retiring the corresponding +link — archival data never changes, so this path is theoretical. + +The existing mapped_variants_clinical_controls association table is left untouched (frozen +for serving existing data). +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "c3d4e5f6a7b8" +down_revision = "b2c3d4e5f6a7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "clinvar_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("clinvar_control_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_clinvar_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["clinvar_control_id"], + ["clinvar_controls.id"], + name="fk_clinvar_allele_links_clinvar_control_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_clinvar_allele_links_allele_id", "clinvar_allele_links", ["allele_id"]) + op.create_index("ix_clinvar_allele_links_clinvar_control_id", "clinvar_allele_links", ["clinvar_control_id"]) + # Multi-live: one live link per (allele, release). An allele accumulates one live link per + # ClinVar release rather than superseding — unlike gnomAD/VEP which enforce one live link + # per allele across all versions. Superseded rows (valid_to IS NOT NULL) are preserved for + # point-in-time queries. + op.create_index( + "uq_clinvar_allele_links_live", + "clinvar_allele_links", + ["allele_id", "clinvar_control_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_clinvar_allele_links_live", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_clinvar_control_id", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_allele_id", table_name="clinvar_allele_links") + op.drop_table("clinvar_allele_links") diff --git a/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py new file mode 100644 index 000000000..26d3d6dbf --- /dev/null +++ b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py @@ -0,0 +1,48 @@ +"""add valid-time versioning to mapping_record_alleles + +Revision ID: c3d5e7f9a1b2 +Revises: b8e1f0a2c4d7 +Create Date: 2026-06-05 + +Make the link table valid-time versioned (TemporalLink): a link is live while valid_to is +NULL, and superseding it closes valid_to instead of deleting, so reverse translation can be +re-run independently while prior derivations remain queryable point-in-time. The partial +unique index enforces a single live link per (mapping_record, allele). + +Assumes no pre-existing duplicate live links — true for these parallel tables, which are +new-only writes and not yet serving. If this ever runs against data with duplicates, the +unique index creation will fail and the duplicates must be retired first. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c3d5e7f9a1b2" +down_revision = "b8e1f0a2c4d7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.add_column( + "mapping_record_alleles", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "uq_mapping_record_alleles_live", + "mapping_record_alleles", + ["mapping_record_id", "allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live", table_name="mapping_record_alleles") + op.drop_column("mapping_record_alleles", "valid_to") + op.drop_column("mapping_record_alleles", "valid_from") diff --git a/alembic/versions/c4b18d0f7a92_add_urn_redirects.py b/alembic/versions/c4b18d0f7a92_add_urn_redirects.py new file mode 100644 index 000000000..043ea825f --- /dev/null +++ b/alembic/versions/c4b18d0f7a92_add_urn_redirects.py @@ -0,0 +1,35 @@ +"""Add urn_redirects, forwarding URNs that publication has retired + +Revision ID: c4b18d0f7a92 +Revises: 393764b9e833 +Create Date: 2026-09-01 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c4b18d0f7a92" +down_revision = "393764b9e833" +branch_labels = None +depends_on = None + + +def upgrade(): + # Not backfilled: publication overwrote each record's temporary URN in place and kept no history of + # it, so the URNs retired before this table existed cannot be recovered. + op.create_table( + "urn_redirects", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("old_urn", sa.String(length=64), nullable=False), + sa.Column("new_urn", sa.String(length=64), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_urn_redirects_old_urn", "urn_redirects", ["old_urn"], unique=True) + + +def downgrade(): + op.drop_index("ix_urn_redirects_old_urn", table_name="urn_redirects") + op.drop_table("urn_redirects") diff --git a/alembic/versions/c5e1a9d4b7f2_add_projection_groups.py b/alembic/versions/c5e1a9d4b7f2_add_projection_groups.py new file mode 100644 index 000000000..b3762678e --- /dev/null +++ b/alembic/versions/c5e1a9d4b7f2_add_projection_groups.py @@ -0,0 +1,75 @@ +"""add projection groups + +Revision ID: c5e1a9d4b7f2 +Revises: b4d8f2a6c1e3 +Create Date: 2026-10-14 + +Makes ``mapping_record_alleles.projection_group`` a foreign key to a ``projection_groups`` row that holds +the group's coding, genomic and protein HGVS. A group's levels need not all have an allele: an intronic +coding projection has no VRS representation but is still the variant's coding coordinate. + +The composite foreign key on ``(projection_group, mapping_record_id)`` keeps a link inside its own record's +groups. Existing group numbers name no row, so they are cleared; re-run the projection step and reverse +translation to rebuild them. Production has none after the substrate backfill, which writes no groups. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c5e1a9d4b7f2" +down_revision = "b4d8f2a6c1e3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "projection_groups", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("mapping_record_id", sa.Integer(), nullable=False), + sa.Column("score_set_id", sa.Integer(), nullable=False), + sa.Column("hgvs_c", sa.String(), nullable=True), + sa.Column("hgvs_g", sa.String(), nullable=True), + sa.Column("hgvs_p", sa.String(), nullable=True), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["mapping_record_id", "score_set_id"], + ["mapping_records.id", "mapping_records.score_set_id"], + name="fk_projection_groups_mapping_record_score_set", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_projection_groups_mapping_record_id", "projection_groups", ["mapping_record_id"]) + # Target of mapping_record_alleles' composite foreign key, which keeps a link inside its own record's groups. + op.create_index( + "uq_projection_groups_id_mapping_record", "projection_groups", ["id", "mapping_record_id"], unique=True + ) + + # Existing group numbers name no row. + mapping_record_alleles = sa.table("mapping_record_alleles", sa.column("projection_group", sa.Integer)) + op.execute( + mapping_record_alleles.update() + .where(mapping_record_alleles.c.projection_group.isnot(None)) + .values(projection_group=None) + ) + op.create_foreign_key( + "fk_mapping_record_alleles_projection_group", + "mapping_record_alleles", + "projection_groups", + ["projection_group", "mapping_record_id"], + ["id", "mapping_record_id"], + ondelete="CASCADE", + ) + op.create_index("ix_mapping_record_alleles_projection_group", "mapping_record_alleles", ["projection_group"]) + + +def downgrade() -> None: + op.drop_index("ix_mapping_record_alleles_projection_group", table_name="mapping_record_alleles") + op.drop_constraint("fk_mapping_record_alleles_projection_group", "mapping_record_alleles", type_="foreignkey") + op.drop_index("uq_projection_groups_id_mapping_record", table_name="projection_groups") + op.drop_index("ix_projection_groups_mapping_record_id", table_name="projection_groups") + op.drop_table("projection_groups") diff --git a/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py new file mode 100644 index 000000000..13c206931 --- /dev/null +++ b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py @@ -0,0 +1,27 @@ +"""index annotation_event.score_set_id + +Revision ID: c9b2a4f8e1d3 +Revises: b8f2c5a1d3e4 +Create Date: 2026-06-30 + +Adds the missing foreign-key index on annotation_event.score_set_id. Every other FK on this table +is indexed; score_set_id was not. The index backs the ON DELETE SET NULL cascade fired when a score +set is deleted (an unindexed FK forces a sequential scan of the event log per deleted score set) and +any operator/BI query that filters the log by score set. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c9b2a4f8e1d3" +down_revision = "b8f2c5a1d3e4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index("ix_annotation_event_score_set_id", "annotation_event", ["score_set_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_annotation_event_score_set_id", table_name="annotation_event") diff --git a/alembic/versions/d1c9b7a3e2f4_add_job_run_id_to_target_gene_mappings.py b/alembic/versions/d1c9b7a3e2f4_add_job_run_id_to_target_gene_mappings.py new file mode 100644 index 000000000..744f3f72d --- /dev/null +++ b/alembic/versions/d1c9b7a3e2f4_add_job_run_id_to_target_gene_mappings.py @@ -0,0 +1,45 @@ +"""add job_run_id to target_gene_mappings + +Revision ID: d1c9b7a3e2f4 +Revises: a3f9c2e1b7d6 +Create Date: 2026-09-02 + +Adds a nullable ``job_run_id`` (FK ``job_runs``, ``ON DELETE SET NULL``) recording the mapping job that +produced each row. Reverse translation uses it to bind a variant's cross-level transcript to its *own* run's +cdna alignment rather than to any same-day run's (#763): the day-granular ``mapped_date`` key it previously +used cannot tell two runs on one calendar day apart, so a same-day remap that emits no cdna row would bind the +earlier run's stale transcript instead of correctly skipping. + +Nullable, no backfill: rows written before this migration keep NULL and fall back to the coarser +``(target_gene_id, mapped_date)`` match. ``SET NULL`` so a pruned job_run degrades to that fallback rather +than dangling. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d1c9b7a3e2f4" +down_revision = "a3f9c2e1b7d6" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("target_gene_mappings", sa.Column("job_run_id", sa.Integer(), nullable=True)) + op.create_foreign_key( + "fk_target_gene_mappings_job_run_id", + "target_gene_mappings", + "job_runs", + ["job_run_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_index("ix_target_gene_mappings_job_run_id", "target_gene_mappings", ["job_run_id"]) + + +def downgrade() -> None: + op.drop_index("ix_target_gene_mappings_job_run_id", table_name="target_gene_mappings") + op.drop_constraint("fk_target_gene_mappings_job_run_id", "target_gene_mappings", type_="foreignkey") + op.drop_column("target_gene_mappings", "job_run_id") diff --git a/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py new file mode 100644 index 000000000..de5758b4c --- /dev/null +++ b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py @@ -0,0 +1,29 @@ +"""add clinvar_variation_id to clinvar_controls + +Revision ID: d4e5f6a7b8c9 +Revises: c3d4e5f6a7b8 +Create Date: 2026-06-22 + +Additive, non-breaking column for ClinVar's canonical public identifier (VariationID), captured forward +from the variant_summary TSV. db_identifier continues to hold the AlleleID (the allele-level handle used +for gnomAD cross-references); this carries the VariationID beside it for eventual external ClinVar links +(clinvar/variation/{id}). Nullable and not yet served — the dedicated clinvar_variants remodel (explicit +fields replacing the generic db_* shape, the serving/UI cutover, and backfill of existing rows) is +deferred. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d4e5f6a7b8c9" +down_revision = "c3d4e5f6a7b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("clinvar_controls", sa.Column("clinvar_variation_id", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("clinvar_controls", "clinvar_variation_id") diff --git a/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py new file mode 100644 index 000000000..603c171a5 --- /dev/null +++ b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py @@ -0,0 +1,76 @@ +"""move mapping_records onto valid-time versioning + +Revision ID: d4e6f8a0b2c3 +Revises: c3d5e7f9a1b2 +Create Date: 2026-06-05 + +Replace the stored `current` flag and the `created_at`/`updated_at` audit dates with valid-time +columns (ValidTime mixin): a mapping record is live while valid_to is NULL, and a re-map retires +the prior version (closing valid_to) instead of flipping a boolean. `current` becomes derived +(valid_to IS NULL). `mapped_date` (the date the mapping was performed) is domain data and stays. + +The partial unique index promotes to the database the "one live mapping record per variant" +invariant the mapping job previously enforced only in app code. + +Backfills from the columns being dropped, so existing rows keep their validity. Assumes no +duplicate live records per variant (true for these pre-cutover parallel tables; otherwise the +unique index creation fails and the duplicates must be retired first). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d4e6f8a0b2c3" +down_revision = "c3d5e7f9a1b2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_records", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True, server_default=sa.func.now()), + ) + op.add_column( + "mapping_records", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + + # Backfill validity from the columns being replaced: a record's life began at created_at, and + # a non-current record was retired at updated_at (the only in-place update it ever took). + op.execute("UPDATE mapping_records SET valid_from = created_at::timestamptz") + op.execute("UPDATE mapping_records SET valid_to = updated_at::timestamptz WHERE current = false") + + op.alter_column("mapping_records", "valid_from", nullable=False) + + op.drop_column("mapping_records", "current") + op.drop_column("mapping_records", "created_at") + op.drop_column("mapping_records", "updated_at") + + op.create_index( + "uq_mapping_records_current", + "mapping_records", + ["variant_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_records_current", table_name="mapping_records") + + op.add_column("mapping_records", sa.Column("current", sa.Boolean(), nullable=True)) + op.add_column("mapping_records", sa.Column("created_at", sa.Date(), nullable=True)) + op.add_column("mapping_records", sa.Column("updated_at", sa.Date(), nullable=True)) + + op.execute("UPDATE mapping_records SET current = (valid_to IS NULL)") + op.execute("UPDATE mapping_records SET created_at = valid_from::date") + op.execute("UPDATE mapping_records SET updated_at = coalesce(valid_to, valid_from)::date") + + op.alter_column("mapping_records", "current", nullable=False) + op.alter_column("mapping_records", "created_at", nullable=False) + op.alter_column("mapping_records", "updated_at", nullable=False) + + op.drop_column("mapping_records", "valid_to") + op.drop_column("mapping_records", "valid_from") diff --git a/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py new file mode 100644 index 000000000..54f3e3d58 --- /dev/null +++ b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py @@ -0,0 +1,71 @@ +"""add gnomad_allele_links table + +Revision ID: e5f7a9c1b3d4 +Revises: d4e6f8a0b2c3 +Create Date: 2026-06-18 + +New valid-time link table connecting deduplicated alleles to gnomAD variants, replacing the frozen +gnomad_variants_mapped_variants association for new-model writes (Step 1 of the annotation +infrastructure migration, docs/design/annotation-infrastructure-migration.md). A link is live while +valid_to is NULL; the partial unique index enforces a single live link per (allele, gnomad variant) +pair. The existing gnomad_variants_mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e5f7a9c1b3d4" +down_revision = "d4e6f8a0b2c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "gnomad_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("gnomad_variant_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_gnomad_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["gnomad_variant_id"], + ["gnomad_variants.id"], + name="fk_gnomad_allele_links_gnomad_variant_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_gnomad_allele_links_allele_id", + "gnomad_allele_links", + ["allele_id"], + ) + op.create_index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_allele_links", + ["gnomad_variant_id"], + ) + # One live link per allele: gnomAD frequency is a single current value, so a version bump + # supersedes the prior link rather than accumulating one live link per version. + op.create_index( + "uq_gnomad_allele_links_live", + "gnomad_allele_links", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_gnomad_allele_links_live", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_gnomad_variant_id", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_allele_id", table_name="gnomad_allele_links") + op.drop_table("gnomad_allele_links") diff --git a/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py new file mode 100644 index 000000000..6d0d0ff7a --- /dev/null +++ b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py @@ -0,0 +1,37 @@ +"""add projection_group to mapping_record_alleles + +Revision ID: e7b2c9a1f4d3 +Revises: f1a3c7e9b2d5 +Create Date: 2026-07-08 + +Adds a nullable ``projection_group`` integer column to ``mapping_record_alleles`` — a within-record +grouping key that pairs the coding and genomic links of one reverse-translation projection pair. +The two members of a pair share a value; the shared protein apex is NULL, as is every row written +before reverse translation runs. See the model and the reverse_translation worker for the full +semantics (per-record index, authoritative fold-in). + +No index is added: the per-URN serving fetch is already covered by +``ix_mapping_record_alleles_mapping_record_id`` and a record's link set is small. No backfill: +existing rows stay NULL and re-group lazily on the next re-map / reverse-translation run. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e7b2c9a1f4d3" +down_revision = "f1a3c7e9b2d5" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("projection_group", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("mapping_record_alleles", "projection_group") diff --git a/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py new file mode 100644 index 000000000..6d7b6c989 --- /dev/null +++ b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py @@ -0,0 +1,46 @@ +"""enforce one live authoritative link per mapping record + +Revision ID: f1a3c7e9b2d5 +Revises: c9b2a4f8e1d3 +Create Date: 2026-07-07 + +Adds a partial unique index enforcing at most one live authoritative link per mapping record. The +existing uq_mapping_record_alleles_live keys on (mapping_record, allele) and so cannot prevent two +different alleles both being flagged is_authoritative for the same record. The serving layer depends +on there being exactly one: the lean whole-set view (lib/score_set_variants) joins the authoritative +link expecting it 1:1 with the variant, so a second live authoritative link would silently duplicate +the variant and double-count its score in the histogram rather than raise. This index moves that +failure to write time in the mapping job, where the bug actually lives. + +Assumes no pre-existing record with two live authoritative links — the invariant is upheld today by +the mapping job's write logic. If this ever runs against data that violates it, the unique index +creation fails and the offending duplicates must be retired first. To check before applying: + + SELECT mapping_record_id, count(*) + FROM mapping_record_alleles + WHERE is_authoritative AND valid_to IS NULL + GROUP BY mapping_record_id HAVING count(*) > 1; +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f1a3c7e9b2d5" +down_revision = "c9b2a4f8e1d3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_alleles", + ["mapping_record_id"], + unique=True, + postgresql_where=sa.text("is_authoritative AND valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live_authoritative", table_name="mapping_record_alleles") diff --git a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py index 12ffd574d..2d377def7 100644 --- a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py +++ b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py @@ -7,23 +7,46 @@ Creates the v_variant_annotations convenience view, which joins variants, mapped_variants, variant_annotation_status, and scoresets into a single flat row per (variant, annotation_type). Intended for operator queries and the variant_annotations CLI script. + +The view SQL is inlined as a frozen literal rather than imported from the model. At this point in +history the view joins the (now legacy) ``mapped_variants`` table; a later migration rewrites it onto +the ``mapping_records`` / ``alleles`` substrate. Importing the live model definition here would make a +fresh replay build *that* later shape at this early revision — before the substrate tables exist — +and fail. Freezing the historical SQL keeps replay green; the rewrite happens in its own migration. """ from alembic import op -from mavedb.db.view import CreateView, DropView -from mavedb.models.variant_annotation_view import definition, signature - # revision identifiers, used by Alembic. revision = "f2a1b3c4d5e6" down_revision = "e3a7b9f1d2c5" branch_labels = None depends_on = None +SIGNATURE = "v_variant_annotations" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + def upgrade() -> None: - op.execute(CreateView(signature, definition, materialized=False)) + op.execute(f"CREATE VIEW {SIGNATURE} AS {DEFINITION}") def downgrade() -> None: - op.execute(DropView(signature, materialized=False)) + op.execute(f"DROP VIEW {SIGNATURE}") diff --git a/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py new file mode 100644 index 000000000..790659d08 --- /dev/null +++ b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py @@ -0,0 +1,49 @@ +"""add cross_level_translation annotation type + +Revision ID: f4d2a9c1b7e3 +Revises: a1b2c3d4e5f6 +Create Date: 2026-06-02 + +Extends ck_variant_annotation_type_valid to allow the 'cross_level_translation' +annotation type. The VRS mapping worker writes one such row per variant to record +whether cross-level translation (filling the levels the assay did not map) +succeeded, was skipped (multivariant / no transcript), or failed. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f4d2a9c1b7e3" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + +_TYPES_OLD = ( + "'vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) +_TYPES_NEW = "'vrs_mapping', 'cross_level_translation', " + ( + "'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) + + +def upgrade() -> None: + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_NEW})", + ) + + +def downgrade() -> None: + op.execute("DELETE FROM variant_annotation_status WHERE annotation_type = 'cross_level_translation'") + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_OLD})", + ) diff --git a/alembic/versions/f6752571d14d_index_variants_by_variant_number.py b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py new file mode 100644 index 000000000..bc8aba1b2 --- /dev/null +++ b/alembic/versions/f6752571d14d_index_variants_by_variant_number.py @@ -0,0 +1,39 @@ +"""Index score set variants by variant number + +Score set CSV queries order rows by the number after '#' in the variant URN (``Variant.variant_number``). Without an +index on that expression, every request sorts the whole score set, and a page deep into a large score set takes +minutes. + +The index evaluates the expression for every row, so every non-null variant URN must end in '#'; an insert +that breaks that fails. No existing URN broke it when this migration was written. + +Building it takes a lock that blocks writes to ``variants``, so in a populated database build it by hand first, which +makes this migration a no-op: + + CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_variants_scoreset_number + ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id); + +Revision ID: f6752571d14d +Revises: 15c40c367731 +Create Date: 2026-09-29 00:00:00.000000 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f6752571d14d" +down_revision = "15c40c367731" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + "CREATE INDEX IF NOT EXISTS ix_variants_scoreset_number " + "ON variants (scoreset_id, (CAST(split_part(urn, '#', 2) AS INTEGER)), id)" + ) + + +def downgrade(): + op.execute("DROP INDEX IF EXISTS ix_variants_scoreset_number") diff --git a/bin/localstack-init.sh b/bin/localstack-init.sh deleted file mode 100755 index 1a00cfcbc..000000000 --- a/bin/localstack-init.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -echo "Initializing local S3 bucket..." -awslocal s3 mb s3://score-set-csv-uploads-dev -echo "S3 bucket 'score-set-csv-uploads-dev' created." \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9835cff4e..99ae52b7d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -10,6 +10,7 @@ services: depends_on: - db - redis + - moto-init env_file: - settings/.env.dev environment: @@ -19,6 +20,7 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ports: - "8002:8000" ulimits: @@ -27,9 +29,10 @@ services: hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo - worker: + worker: &worker image: mavedb-api/mavedb-worker:dev build: context: . @@ -44,16 +47,25 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ulimits: nofile: soft: 65536 hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo depends_on: - db - redis + - moto-init + + # Runs only start_pipeline and create_variants_for_score_set (the fast-lane queue), as in production, so + # variant creation never waits behind mapping jobs on the main worker. + fast-worker: + <<: *worker + command: bash -c "arq mavedb.worker.FastLaneWorkerSettings --watch ./src" dcd-mapping: build: ../dcd_mapping @@ -86,13 +98,13 @@ services: - mavedb-seqrepo-dev:/usr/local/share/seqrepo db: - image: postgres:14 + image: postgres:15 env_file: - settings/.env.dev ports: - "5434:5432" volumes: - - mavedb-data-dev:/var/lib/postgresql/data + - mavedb-data-dev-15:/var/lib/postgresql/data redis: image: redis:7.2.3 @@ -103,17 +115,19 @@ services: volumes: - mavedb-redis-dev:/data - localstack: - image: localstack/localstack:latest + # AWS mock. + moto: + image: motoserver/moto:latest ports: - - "4566:4566" - env_file: - - settings/.env.dev - environment: - - SERVICES=s3:4566 # We only need S3 for MaveDB - volumes: - - mavedb-localstack-dev:/var/lib/localstack - - "./bin/localstack-init.sh:/etc/localstack/init/ready.d/localstack-init.sh" + - "5001:5000" + + moto-init: + # One-shot: create the upload bucket once moto is up (moto has no auto-create). + # A raw PUT is all it takes; --retry-all-errors waits out moto's startup. + image: curlimages/curl:latest + depends_on: + - moto + command: -sf --retry 10 --retry-all-errors --retry-delay 1 -X PUT http://moto:5000/score-set-csv-uploads-dev seqrepo: image: biocommons/seqrepo:2024-12-20 @@ -121,7 +135,6 @@ services: - mavedb-seqrepo-dev:/usr/local/share/seqrepo volumes: - mavedb-data-dev: + mavedb-data-dev-15: mavedb-redis-dev: mavedb-seqrepo-dev: - mavedb-localstack-dev: diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 69a9fd1a1..baa5e9bd0 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -23,7 +23,7 @@ services: - "8444:8000" restart: unless-stopped - worker: + worker: &worker image: mavedb-api-worker/mavedb-api-worker:latest build: context: . @@ -43,15 +43,21 @@ services: - redis restart: unless-stopped + # Runs only start_pipeline and create_variants_for_score_set (the fast-lane queue), as in production, so + # variant creation never waits behind mapping jobs on the main worker. + fast-worker: + <<: *worker + command: bash -c "arq src.mavedb.worker.FastLaneWorkerSettings" + db: - image: postgres:14 + image: postgres:15 env_file: - settings/.env.dev restart: unless-stopped ports: - "5434:5432" volumes: - - mavedb-data:/var/lib/postgresql/data + - mavedb-data-15:/var/lib/postgresql/data redis: image: redis:7.2.3 @@ -68,5 +74,5 @@ services: # - "5673:5672" volumes: - mavedb-data: + mavedb-data-15: mavedb-redis: diff --git a/mypy_stubs/ga4gh/core/__init__.pyi b/mypy_stubs/ga4gh/core/__init__.pyi deleted file mode 100644 index f5814d7cf..000000000 --- a/mypy_stubs/ga4gh/core/__init__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -from . import models as models - -__all__ = ["models"] diff --git a/mypy_stubs/ga4gh/core/identifiers.pyi b/mypy_stubs/ga4gh/core/identifiers.pyi deleted file mode 100644 index 04fddede0..000000000 --- a/mypy_stubs/ga4gh/core/identifiers.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from enum import Enum -from re import Pattern - -GA4GH_IR_REGEXP: Pattern[str] - -class PrevVrsVersion(str, Enum): - V1_3 = "1.3" - - @classmethod - def validate(cls, version) -> None: ... diff --git a/mypy_stubs/ga4gh/core/models.pyi b/mypy_stubs/ga4gh/core/models.pyi deleted file mode 100644 index 2acc13a5e..000000000 --- a/mypy_stubs/ga4gh/core/models.pyi +++ /dev/null @@ -1,55 +0,0 @@ -from enum import Enum - -from pydantic import BaseModel, RootModel -from typing import Annotated, Any - -class Relation(str, Enum): - CLOSE_MATCH = "closeMatch" - EXACT_MATCH = "exactMatch" - BROAD_MATCH = "broadMatch" - NARROW_MATCH = "narrowMatch" - RELATED_MATCH = "relatedMatch" - -class Element(BaseModel): - id: str | None = None - extensions: list[Extension] | None = None - -class Code(RootModel): - root: Annotated[str, None] - -class Coding(BaseModel): - label: str | None = None - system: str - systemVersion: str | None = None - code: Code - -class Extension(Element): - name: str - value: float | str | bool | dict[str, Any] | list[Any] | None = None - description: str | None = None - -class MappableConcept(BaseModel): - name: str | None = None - conceptType: str | None = None - primaryCoding: Coding | None = None - mappings: list[ConceptMapping] | None = None - - def require_name_or_primary_coding(cls, v): ... - -class Entity(BaseModel): - id: str | None = None - type: str - name: str | None = None - description: str | None = None - aliases: list[str] | None = None - extensions: list[Extension] | None = None - -class iriReference(RootModel): # noqa: N801 - def __hash__(self) -> int: ... - def ga4gh_serialize(self) -> str: ... - - root: str - -class ConceptMapping(Element): - coding: Coding - relation: Relation diff --git a/mypy_stubs/ga4gh/va_spec/acmg_2015.pyi b/mypy_stubs/ga4gh/va_spec/acmg_2015.pyi deleted file mode 100644 index 1e5e8b573..000000000 --- a/mypy_stubs/ga4gh/va_spec/acmg_2015.pyi +++ /dev/null @@ -1,51 +0,0 @@ -from enum import Enum - -from ..core.models import MappableConcept -from .base.core import EvidenceLine, Method, Statement, VariantPathogenicityProposition, iriReference - -class VariantPathogenicityEvidenceLine(EvidenceLine): - targetProposition: VariantPathogenicityProposition | None # type: ignore - strengthOfEvidenceProvided: MappableConcept | None - specifiedBy: Method | iriReference | None - - class Criterion(str, Enum): - """Define ACMG 2015 criterion values""" - - PVS1 = "PVS1" - PS1 = "PS1" - PS2 = "PS2" - PS3 = "PS3" - PS4 = "PS4" - PM1 = "PM1" - PM2 = "PM2" - PM3 = "PM3" - PM4 = "PM4" - PM5 = "PM5" - PM6 = "PM6" - PP1 = "PP1" - PP2 = "PP2" - PP3 = "PP3" - PP4 = "PP4" - PP5 = "PP5" - BA1 = "BA1" - BS1 = "BS1" - BS2 = "BS2" - BS3 = "BS3" - BS4 = "BS4" - BP1 = "BP1" - BP2 = "BP2" - BP3 = "BP3" - BP4 = "BP4" - BP5 = "BP5" - BP6 = "BP6" - BP7 = "BP7" - -class VariantPathogenicityStatement(Statement): - proposition: VariantPathogenicityProposition - -class AcmgClassification(str, Enum): - PATHOGENIC = "pathogenic" - BENIGN = "benign" - LIKELY_PATHOGENIC = "likely pathogenic" - LIKELY_BENIGN = "likely benign" - UNCERTAIN_SIGNIFICANCE = "uncertain significance" diff --git a/mypy_stubs/ga4gh/va_spec/base/__init__.pyi b/mypy_stubs/ga4gh/va_spec/base/__init__.pyi deleted file mode 100644 index 5fcb5c274..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from . import core as core -from . import enums as enums - -__all__ = ["core", "enums"] diff --git a/mypy_stubs/ga4gh/va_spec/base/core.pyi b/mypy_stubs/ga4gh/va_spec/base/core.pyi deleted file mode 100644 index 2fd15c263..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/core.pyi +++ /dev/null @@ -1,193 +0,0 @@ -from _typeshed import Incomplete -from abc import ABC -from datetime import date -from enum import Enum -from pydantic import BaseModel, RootModel, StringConstraints as StringConstraints -from typing import Annotated, Any, Literal, TypeVar - -from ...core.models import Coding, MappableConcept -from ...vrs.models import MolecularVariation - -StatementType = TypeVar("StatementType") -EvidenceLineType = TypeVar("EvidenceLineType") - -class CoreImType(str, Enum): - AGENT = str - CONTRIBUTION = str - DOCUMENT = str - METHOD = str - DATA_SET = str - EVIDENCE_LINE = str - INFORMATION_ENTITY = str - STUDY_GROUP = str - -class Direction(str, Enum): - SUPPORTS = str - NEUTRAL = str - DISPUTES = str - -class Code(RootModel): - root: Annotated[str, None] - -class iriReference(RootModel): - def __hash__(self): ... - def ga4gh_serialize(self): ... - root: str - -class Extension(BaseModel): - name: str - value: float | str | bool | dict[str, Any] | list[Any] | None = None - description: str | None = None - -class Entity(BaseModel, ABC): - id: str | None = None - type: str - label: str | None = None - description: str | None = None - alternativeLabels: list[str] | None = None - extensions: list[Extension] | None = None - -class Agent(Entity): - type: Literal["Agent"] = "Agent" - name: str | None = None - -class ActivityBase(Entity, ABC): - subtype: Coding | None = None - date: str | None = None - specifiedBy: list[Method] | None = None - @classmethod - def date_format(cls, v: str | None) -> str | None: ... - -class Activity(ActivityBase): - performedBy: list[Agent] | None = None - -class Contribution(ActivityBase): - type: Literal["Contribution"] = "Contribution" - contributor: list[Agent] | None = None - activityType: Coding | None = None - -class InformationEntityBase(Entity, ABC): - type: Literal["InformationEntity"] = "InformationEntity" - specifiedBy: Method | iriReference | None = None - contributions: list[Contribution] | None = None - reportedIn: list[Document | iriReference] | None = None - dateAuthored: str | None = None - recordMetadata: RecordMetadata | None = None - -class InformationEntity(InformationEntityBase): - derivedFrom: list[InformationEntity] | None = None - -class Document(InformationEntity): - type: Literal["Document"] = "Document" - subtype: Coding | None = None - title: str | None = None - urls: list[Annotated[str, None]] | None = None - doi: Annotated[str, None] | None = None - pmid: int | None = None - -class Method(InformationEntity): - type: Literal["Method"] = "Method" - subtype: Coding | None = None - license: str | None = None - -class RecordMetadata(BaseModel): - recordIdentifier: str | None = None - recordVersion: str | None = None - derivedFrom: str | None = None - dateRecordCreated: str | None = None - contributions: list[Contribution] | None = None - -class DataSet(Entity): - type: Literal["DataSet"] = "DataSet" - subtype: MappableConcept | None = None - reportedIn: Document | iriReference | None = None - releaseDate: date | None = None - version: str | None = None - license: MappableConcept | None = None - -class EvidenceLine(InformationEntity): - type: Literal["EvidenceLine"] = "EvidenceLine" - targetProposition: Proposition | SubjectVariantProposition | None = None - hasEvidenceItems: list[StudyResult | Statement | EvidenceLine | iriReference] | None = None - directionOfEvidenceProvided: Direction - strengthOfEvidenceProvided: MappableConcept | None = None - scoreOfEvidenceProvided: float | None = None - evidenceOutcome: MappableConcept | None = None - -class Proposition(Entity): - subject: dict - predicate: str - object: dict - -class Statement(InformationEntity): - type: Literal["Statement"] = "Statement" - proposition: ( - ExperimentalVariantFunctionalImpactProposition - | VariantDiagnosticProposition - | VariantOncogenicityProposition - | VariantPathogenicityProposition - | VariantPrognosticProposition - | VariantTherapeuticResponseProposition - ) - direction: Direction - strength: MappableConcept | None = None - score: float | None = None - classification: MappableConcept | None = None - hasEvidenceLines: list[EvidenceLine | iriReference] | None = None - -class StudyResult(RootModel): - root: CohortAlleleFrequencyStudyResult | ExperimentalVariantFunctionalImpactStudyResult - -class _StudyResult(InformationEntity, ABC): - sourceDataSet: DataSet | None = None - ancillaryResults: dict | None = None - qualityMeasures: dict | None = None - -class _SubjectVariantPropositionBase(Entity, ABC): - subjectVariant: MolecularVariation | CategoricalVariant | iriReference - -class SubjectVariantProposition(RootModel): - root: ( - ExperimentalVariantFunctionalImpactProposition - | VariantPathogenicityProposition - | VariantDiagnosticProposition - | VariantPrognosticProposition - | VariantOncogenicityProposition - | VariantTherapeuticResponseProposition - ) - -class ClinicalVariantProposition(_SubjectVariantPropositionBase): - geneContextQualifier: MappableConcept | iriReference | None = None - alleleOriginQualifier: MappableConcept | iriReference | None = None - -class VariantPathogenicityProposition(ClinicalVariantProposition): - type: Literal["VariantPathogenicityProposition"] = "VariantPathogenicityProposition" - targetProposition: VariantPathogenicityProposition | None = None - strengthOfEvidenceProvided: Coding | iriReference | None = None - specifiedBy: Method | iriReference | None = None - hasEvidenceLines: list[EvidenceLine] | None = None - -class ExperimentalVariantFunctionalImpactStudyResult(_StudyResult): - type: Literal["ExperimentalVariantFunctionalImpactStudyResult"] = "ExperimentalVariantFunctionalImpactStudyResult" - focusVariant: MolecularVariation | iriReference - functionalImpactScore: float | None = None - specifiedBy: Method | iriReference | None = None - sourceDataSet: DataSet | None = None - -class ExperimentalVariantFunctionalImpactProposition(_SubjectVariantPropositionBase): - type: Literal["ExperimentalVariantFunctionalImpactProposition"] = "ExperimentalVariantFunctionalImpactProposition" - predicate: str = "impactsFunctionOf" - objectSequenceFeature: iriReference | MappableConcept - experimentalContextQualifier: iriReference | Document | dict | None = None - -# TODO#319: When we pull in CatVRS as a dependency, we can fully type this but as of now there is no need. -CategoricalVariant = Incomplete - -# Proposition types we don't care about and thus have no need to strongly type. -VariantDiagnosticProposition = Incomplete -VariantOncogenicityProposition = Incomplete -VariantPrognosticProposition = Incomplete -VariantTherapeuticResponseProposition = Incomplete - -# Study result types we don't care about and thus have no need to strongly type. -CohortAlleleFrequencyStudyResult = Incomplete diff --git a/mypy_stubs/ga4gh/va_spec/base/domain_entities.pyi b/mypy_stubs/ga4gh/va_spec/base/domain_entities.pyi deleted file mode 100644 index 10b57d368..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/domain_entities.pyi +++ /dev/null @@ -1,11 +0,0 @@ -from pydantic import RootModel - -from ...core.models import Element, MappableConcept -from .enums import MembershipOperator - -class ConditionSet(Element): - conditions: list[MappableConcept] - membershipOperator: MembershipOperator - -class Condition(RootModel): - root: ConditionSet | MappableConcept diff --git a/mypy_stubs/ga4gh/va_spec/base/enums.pyi b/mypy_stubs/ga4gh/va_spec/base/enums.pyi deleted file mode 100644 index 04adba73c..000000000 --- a/mypy_stubs/ga4gh/va_spec/base/enums.pyi +++ /dev/null @@ -1,12 +0,0 @@ -from enum import Enum - -class MembershipOperator(str, Enum): - AND = "AND" - OR = "OR" - -class StrengthOfEvidenceProvided(str, Enum): - STANDALONE = "standalone" - VERY_STRONG = "very strong" - STRONG = "strong" - MODERATE = "moderate" - SUPPORTING = "supporting" diff --git a/mypy_stubs/ga4gh/vrs/__init__.pyi b/mypy_stubs/ga4gh/vrs/__init__.pyi deleted file mode 100644 index 3bafe7db9..000000000 --- a/mypy_stubs/ga4gh/vrs/__init__.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from . import models as models -from .models import VrsType as VrsType - -__all__ = ["VrsType", "models"] diff --git a/mypy_stubs/ga4gh/vrs/models.pyi b/mypy_stubs/ga4gh/vrs/models.pyi deleted file mode 100644 index 73c491d88..000000000 --- a/mypy_stubs/ga4gh/vrs/models.pyi +++ /dev/null @@ -1,230 +0,0 @@ -from _typeshed import Incomplete -from abc import ABC -from collections.abc import Generator -from enum import Enum -from ..core.identifiers import PrevVrsVersion as PrevVrsVersion -from ..core.models import Entity as Entity, iriReference as IRI -from pydantic import BaseModel, RootModel, StringConstraints as StringConstraints, ValidationInfo as ValidationInfo -from typing import Annotated, Literal - -def flatten(vals) -> Generator[Incomplete, None, Incomplete]: ... -def flatten_type(t): ... -def overlaps(a: list, b: list): ... -def pydantic_class_refatt_map(): ... - -class VrsType(str, Enum): - LEN_EXPR = "LengthExpression" - REF_LEN_EXPR = "ReferenceLengthExpression" - LIT_SEQ_EXPR = "LiteralSequenceExpression" - SEQ_REF = "SequenceReference" - SEQ_LOC = "SequenceLocation" - ALLELE = "Allele" - CIS_PHASED_BLOCK = "CisPhasedBlock" - ADJACENCY = "Adjacency" - TERMINUS = "Terminus" - TRAVERSAL_BLOCK = "TraversalBlock" - DERIVATIVE_MOL = "DerivativeMolecule" - CN_COUNT = "CopyNumberCount" - CN_CHANGE = "CopyNumberChange" - -class Orientation(str, Enum): - FORWARD = "forward" - REVERSE_COMPLEMENT = "reverse_complement" - -class ResidueAlphabet(str, Enum): - AA = "aa" - NA = "na" - -class CopyChange(str, Enum): - COMPLETE_GENOMIC_LOSS = "complete genomic loss" - HIGH_LEVEL_LOSS = "high-level loss" - LOW_LEVEL_LOSS = "low-level loss" - LOSS = "loss" - REGIONAL_BASE_PLOIDY = "regional base ploidy" - GAIN = "gain" - LOW_LEVEL_GAIN = "low-level gain" - HIGH_LEVEL_GAIN = "high-level gain" - -class Syntax(str, Enum): - HGVS_C = "hgvs.c" - HGVS_P = "hgvs.p" - HGVS_G = "hgvs.g" - HGVS_M = "hgvs.m" - HGVS_N = "hgvs.n" - HGVS_R = "hgvs.r" - HGVS_ISCN = "iscn" - GNOMAD = "gnomad" - SPDI = "spdi" - -class _ValueObject(Entity, ABC): - def __hash__(self): ... - def ga4gh_serialize(self) -> dict: ... - class ga4gh: - keys: list[str] - - @staticmethod - def is_ga4gh_identifiable(): ... - -class _Ga4ghIdentifiableObject(_ValueObject, ABC): - type: str - digest: Annotated[str, None] | None = None - def __lt__(self, other): ... - @staticmethod - def is_ga4gh_identifiable(): ... - def has_valid_ga4gh_id(self): ... - def compute_digest(self, store: bool = True, as_version: PrevVrsVersion | None = None) -> str: ... - id: Incomplete - def get_or_create_ga4gh_identifier( - self, in_place: str = "default", recompute: bool = False, as_version: Incomplete | None = None - ) -> str: ... - def compute_ga4gh_identifier(self, recompute: bool = False, as_version: Incomplete | None = None): ... - def get_or_create_digest(self, recompute: bool = False) -> str: ... - class ga4gh(_ValueObject.ga4gh): - prefix: str - -class Expression(BaseModel): - syntax: Syntax - value: str - syntax_version: str | None = None - -class Range(RootModel): - root: list[int | None] - def validate_range(cls, v: list[int | None]) -> list[int | None]: ... - -class Residue(RootModel): - root: Annotated[str, None] - -class SequenceString(RootModel): - root: Annotated[str, None] - -class LengthExpression(_ValueObject): - type: Literal["LengthExpression"] = "LengthExpression" - length: Range | int | None = None - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class ReferenceLengthExpression(_ValueObject): - type: Literal["ReferenceLengthExpression"] = "ReferenceLengthExpression" - length: Range | int - sequence: SequenceString | None = None - repeatSubunitLength: int - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class LiteralSequenceExpression(_ValueObject): - type: Literal["LiteralSequenceExpression"] = "LiteralSequenceExpression" - sequence: SequenceString - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class SequenceReference(_ValueObject): - type: Literal["SequenceReference"] = "SequenceReference" - refgetAccession: Annotated[str, None] - residueAlphabet: ResidueAlphabet | None = None - circular: bool | None = None - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class SequenceLocation(_Ga4ghIdentifiableObject): - type: Literal["SequenceLocation"] = "SequenceLocation" - sequenceReference: IRI | SequenceReference | None = None - start: Range | int | None = None - end: Range | int | None = None - sequence: SequenceString | None = None - def validate_start_end(cls, v: Range | int | None, info: ValidationInfo) -> Range | int | None: ... - def ga4gh_serialize_as_version(self, as_version: PrevVrsVersion): ... - def get_refget_accession(self): ... - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - priorPrefix: Incomplete - keys: Incomplete - -class _VariationBase(_Ga4ghIdentifiableObject, ABC): - expressions: list[Expression] | None = None - -class Allele(_VariationBase): - type: Literal["Allele"] = "Allele" - location: IRI | SequenceLocation - state: LiteralSequenceExpression | ReferenceLengthExpression | LengthExpression - def ga4gh_serialize_as_version(self, as_version: PrevVrsVersion): ... - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - priorPrefix: Incomplete - keys: Incomplete - -class CisPhasedBlock(_VariationBase): - type: Literal["CisPhasedBlock"] = "CisPhasedBlock" - members: list[Allele | IRI] - sequenceReference: SequenceReference | None = None - def ga4gh_serialize(self) -> dict: ... - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class Adjacency(_VariationBase): - type: Literal["Adjacency"] = "Adjacency" - adjoinedSequences: list[IRI | SequenceLocation] - linker: LiteralSequenceExpression | ReferenceLengthExpression | LengthExpression | None = None - homology: bool | None = None - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class Terminus(_VariationBase): - type: Literal["Terminus"] = "Terminus" - location: IRI | SequenceLocation - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class TraversalBlock(_ValueObject): - type: Literal["TraversalBlock"] = "TraversalBlock" - orientation: Orientation | None = None - component: Allele | CisPhasedBlock | Adjacency | Terminus | None = None - class ga4gh(_ValueObject.ga4gh): - keys: Incomplete - -class DerivativeMolecule(_VariationBase): - type: Literal["DerivativeMolecule"] = "DerivativeMolecule" - components: list[IRI | TraversalBlock] - circular: bool | None = None - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class _CopyNumber(_VariationBase, ABC): - location: IRI | SequenceLocation - -class CopyNumberCount(_CopyNumber): - type: Literal["CopyNumberCount"] = "CopyNumberCount" - copies: Range | int - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class CopyNumberChange(_CopyNumber): - type: Literal["CopyNumberChange"] = "CopyNumberChange" - copyChange: CopyChange - class ga4gh(_Ga4ghIdentifiableObject.ga4gh): - prefix: str - keys: Incomplete - -class MolecularVariation(RootModel): - root: Allele | CisPhasedBlock | Adjacency | Terminus | DerivativeMolecule - -class SequenceExpression(RootModel): - root: LiteralSequenceExpression | ReferenceLengthExpression | LengthExpression - -class Location(RootModel): - root: SequenceLocation - -class Variation(RootModel): - root: Allele | CisPhasedBlock | Adjacency | Terminus | DerivativeMolecule | CopyNumberChange | CopyNumberCount - -class SystemicVariation(RootModel): - root: CopyNumberChange | CopyNumberCount - -reffable_classes: Incomplete -union_reffable_classes: Incomplete -class_refatt_map: Incomplete -class_keys: Incomplete diff --git a/mypy_stubs/hgvs/assemblymapper.pyi b/mypy_stubs/hgvs/assemblymapper.pyi new file mode 100644 index 000000000..798878312 --- /dev/null +++ b/mypy_stubs/hgvs/assemblymapper.pyi @@ -0,0 +1,20 @@ +from typing import Any + +import hgvs.sequencevariant + +class AssemblyMapper: + def __init__( + self, + hdp: Any, + assembly_name: str = ..., + alt_aln_method: str = ..., + normalize: bool = ..., + prevalidation_level: str | None = ..., + in_par_assume: str = ..., + replace_reference: bool = ..., + *args: Any, + **kwargs: Any, + ) -> None: ... + def g_to_t(self, var_g: Any, tx_ac: str) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_g(self, var_c: Any) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_p(self, var_c: Any, translation_table: Any = ...) -> hgvs.sequencevariant.SequenceVariant: ... diff --git a/poetry.lock b/poetry.lock index 93760a9d5..8455104b2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -93,19 +93,19 @@ files = [ [[package]] name = "anyio" -version = "4.13.0" +version = "4.15.1" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708"}, - {file = "anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc"}, + {file = "anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101"}, + {file = "anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94"}, ] [package.dependencies] idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} +typing_extensions = {version = ">=4.16.0", markers = "python_version < \"3.15\""} [package.extras] trio = ["trio (>=0.32.0)"] @@ -269,6 +269,61 @@ dev = ["bandit (>=1.7,<2.0)", "build (>=0.8,<1.0)", "flake8 (>=4.0,<5.0)", "ipyt docs = ["mkdocs"] tests = ["pytest (>=7.1,<8.0)", "pytest-cov (>=4.1,<5.0)", "pytest-optional-tests", "tox (>=3.25,<4.0)", "vcrpy"] +[[package]] +name = "biopython" +version = "1.88" +description = "Freely available tools for computational molecular biology." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "biopython-1.88-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9732e2c74b1e3751810b026e568817afb18101039c30c9b16b9193e40ec1efdb"}, + {file = "biopython-1.88-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02a6d8ed4eea1ee9e1c2a9d7690f21a60406bcfee7c71348363e267cce397bee"}, + {file = "biopython-1.88-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b321ee0f652aaecefe065b11d64b527c4d04f7826dfb7cfa31edea5ab1db07c2"}, + {file = "biopython-1.88-cp310-cp310-win32.whl", hash = "sha256:99288eb34ce75cb85a10ab3e5d76ffe45f0c03c7f5cc6d144d3777419e44b31e"}, + {file = "biopython-1.88-cp310-cp310-win_amd64.whl", hash = "sha256:0e732f44208f1bab7c42bfea94422609249bd897da6b25b5588618436e80a6d2"}, + {file = "biopython-1.88-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:793de1a57d9b75546a4041f547b6a848031a71933f66d4c761857b0b0118c46b"}, + {file = "biopython-1.88-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:906a7b2d4bc4515f5718f3ae994bd3bcfe8e87d963954008b57e0109e3772a58"}, + {file = "biopython-1.88-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99e1f7022ab8eee2ae9e9a05b66ad183b9fe4eeb4e561a65ed0fd85eef5f8267"}, + {file = "biopython-1.88-cp311-cp311-win32.whl", hash = "sha256:024e438d044a0b1d4a0157af4ab2411b83389696aee3cdeb71f5ee74d0a59df2"}, + {file = "biopython-1.88-cp311-cp311-win_amd64.whl", hash = "sha256:5808c5b980e63226426737b6f71b5dd7747e2e3d0074720cc04edf348ab57eab"}, + {file = "biopython-1.88-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:43ce1fbed01c0ddb903fa426fa0f836062d973a09df15560ac8e895da24c1350"}, + {file = "biopython-1.88-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe2dccf88e309dc896e7c2cc974a47be7a7b96d59c7f2ffd74ffcc7cfbdebb0c"}, + {file = "biopython-1.88-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9384c30460832926e765319040d732d1252a713a05af1dd4cc8b19cb1d590439"}, + {file = "biopython-1.88-cp312-cp312-win32.whl", hash = "sha256:4904abb18a1e039a631e549529c9b8123a166838cafe27f649f83f0183382d61"}, + {file = "biopython-1.88-cp312-cp312-win_amd64.whl", hash = "sha256:dd5131b70287486af1acef2fc40de51c68de2ea740ab772097ac2d10bad9a0b5"}, + {file = "biopython-1.88-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:465caece761b9ec22e0dc63bb5969517c901587a27104c4f1cee69f414b4df2d"}, + {file = "biopython-1.88-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df709eabdce2fc2cc1059fcefbbea245d2b2c41fc2026eda6194b810474e68d6"}, + {file = "biopython-1.88-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:410eaa083fcc308b44c7051e597ae2f39e4068754bdd605cc32aed025aa132d0"}, + {file = "biopython-1.88-cp313-cp313-win32.whl", hash = "sha256:e3ac69b8abddac75dce301cbe77fc2650cbe0bd62ab2d2662d0f6f378b8c8018"}, + {file = "biopython-1.88-cp313-cp313-win_amd64.whl", hash = "sha256:f0aaf49f0c8919fb7a02cdc3db9e7f89165f53417a18d80508e0f167121cb7c1"}, + {file = "biopython-1.88-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5508f8f585d101269982d58abb4ab50cbde5d3c50e21b01986c55ac7cc5f34f9"}, + {file = "biopython-1.88-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ffa3f911a8ded827663bb98ffcd5fcfcaed2c7262bf6d3be5338332c81c7b66"}, + {file = "biopython-1.88-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99144485e7723842fe6c6bb0c13f323a550fc30817d3e0814126987f9b06855a"}, + {file = "biopython-1.88-cp314-cp314-win32.whl", hash = "sha256:b86c1e86c829b372995d5e0eb80146551972a1bff25da1698b48844b6e6840cd"}, + {file = "biopython-1.88-cp314-cp314-win_amd64.whl", hash = "sha256:0b8fea4940c58915e6d4c670f17f4e9c20797e9e26d4300b6efbe960b99d199d"}, + {file = "biopython-1.88-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:393398c8383217f3f51aaa2372960e697b494a3eee97e1c4d5f62ee60ea9287b"}, + {file = "biopython-1.88-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab5c2211a9d1ad7d4b8afd7f53caa7f20e104a5ddf0b30f7cb899f039379de68"}, + {file = "biopython-1.88-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:adbafdc7769d46cc9e72aba86b7e2da3aa078d2a533b53757ca88ca6ff1d0489"}, + {file = "biopython-1.88-cp314-cp314t-win32.whl", hash = "sha256:1e29b72bb79786f4157b394fbd852d736ffd883a13a177c21d6d15ea1b67d357"}, + {file = "biopython-1.88-cp314-cp314t-win_amd64.whl", hash = "sha256:f684b0fddbfe3b37de36d1b377c4c10d968226940c307a5a16e360e63063900f"}, + {file = "biopython-1.88-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:f238b612f15166c6a3c869f833b5aa6c0dd448dabb1d4b94b132281de3e91cab"}, + {file = "biopython-1.88-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c5c795fed9469b1115fd00d7681f4953a2d54ec502715720d11fe986678fa1b"}, + {file = "biopython-1.88-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e6e675ff0ecd0822735c9da0b7091eb29a757279bc810039890ae1f54f18fdd"}, + {file = "biopython-1.88-cp315-cp315-win32.whl", hash = "sha256:277b54f1b804a74df1eb7f0fe8f6d6a1eda15bc78a94802c000f5f0a20ce4fdc"}, + {file = "biopython-1.88-cp315-cp315-win_amd64.whl", hash = "sha256:d4180714488d2ac6418fd42c445d1cda71fdd85ff71ad6748e7a312fa944e138"}, + {file = "biopython-1.88-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:e800cacbf8e240105338f8eb52e92a00cd0103881d3cdf1798e89130cf878ed8"}, + {file = "biopython-1.88-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f300ef2db3d3fd33631364fe3f83a64d281d255dd0e8182feb8e3db9e290ce32"}, + {file = "biopython-1.88-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0063aea2051a99b2f278dd8bef23d5eca3f50eb922fb72d4a99678484bae043d"}, + {file = "biopython-1.88-cp315-cp315t-win32.whl", hash = "sha256:d879c732e0dee158cebf09047fa616451fa30f3abcab4105e3ce5cd336571147"}, + {file = "biopython-1.88-cp315-cp315t-win_amd64.whl", hash = "sha256:abeda1d57624b82476be498259dbc4531a1957bb1d77c80fd986adaa44f1b406"}, + {file = "biopython-1.88.tar.gz", hash = "sha256:9aaa31c0bda4d059f7b2ee00bfdb5cbb73ade3057aa4b737a7cc0187091d071a"}, +] + +[package.dependencies] +numpy = "*" + [[package]] name = "bioutils" version = "0.6.1" @@ -1279,76 +1334,79 @@ toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "cryptography" -version = "46.0.7" +version = "50.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = true -python-versions = "!=3.9.0,!=3.9.1,>=3.8" +python-versions = "!=3.9.0,!=3.9.1,>=3.9" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb"}, - {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b"}, - {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85"}, - {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e"}, - {file = "cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457"}, - {file = "cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b"}, - {file = "cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1"}, - {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2"}, - {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e"}, - {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee"}, - {file = "cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298"}, - {file = "cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb"}, - {file = "cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006"}, - {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0"}, - {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85"}, - {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e"}, - {file = "cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246"}, - {file = "cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968"}, - {file = "cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4"}, - {file = "cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5"}, + {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"}, + {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"}, + {file = "cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e"}, + {file = "cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c"}, + {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"}, + {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452"}, + {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"}, ] [package.dependencies] -cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] -docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox[uv] (>=2024.4.15)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] -sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] -test-randomorder = ["pytest-randomly"] [[package]] name = "decorator" @@ -1432,6 +1490,19 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + [[package]] name = "eutils" version = "0.6.0" @@ -1511,27 +1582,30 @@ probabilistic = ["pyprobables (>=0.6,<0.7)"] [[package]] name = "fastapi" -version = "0.121.3" +version = "0.142.2" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = true -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "fastapi-0.121.3-py3-none-any.whl", hash = "sha256:0c78fc87587fcd910ca1bbf5bc8ba37b80e119b388a7206b39f0ecc95ebf53e9"}, - {file = "fastapi-0.121.3.tar.gz", hash = "sha256:0055bc24fe53e56a40e9e0ad1ae2baa81622c406e548e501e717634e2dfbc40b"}, + {file = "fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b"}, + {file = "fastapi-0.142.2.tar.gz", hash = "sha256:06366626f2e70576367714d9ab2fe8472e6c8456dba69b399f9f797ab5e92570"}, ] [package.dependencies] annotated-doc = ">=0.0.2" -pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" -starlette = ">=0.40.0,<0.51.0" +opentelemetry-api = ">=1.44.0" +pydantic = ">=2.9.0" +starlette = ">=0.46.0" typing-extensions = ">=4.8.0" +typing-inspection = ">=0.4.2" [package.extras] -all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] -standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] -standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] +all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.32)", "httpx (>=0.23.0,<1.0.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "uvicorn[standard] (>=0.12.0)"] +opentelemetry = ["opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)"] +standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.32)", "fastar (>=0.9.0)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] +standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.32)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "opentelemetry-exporter-otlp-proto-http (>=1.44.0)", "opentelemetry-sdk (>=1.44.0)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] [[package]] name = "filelock" @@ -1619,18 +1693,18 @@ tqdm = ["tqdm"] [[package]] name = "ga4gh-cat-vrs" -version = "0.7.2" +version = "0.8.0a6" description = "GA4GH Categorical Variation Representation (Cat-VRS) reference implementation" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ga4gh_cat_vrs-0.7.2-py3-none-any.whl", hash = "sha256:153da80cbe305a4a2b81d9d1576f7c9fc85eb110f7a80f0e708ed9f7358925c8"}, - {file = "ga4gh_cat_vrs-0.7.2.tar.gz", hash = "sha256:f06530d28aa21bf007ce0cfeee4a98458fc6506ce552029866414fd303fa4de6"}, + {file = "ga4gh_cat_vrs-0.8.0a6-py3-none-any.whl", hash = "sha256:3d9fb48dda3bb4ddb250e042716d0aaa679ee963e281f9609de6d979a774941c"}, + {file = "ga4gh_cat_vrs-0.8.0a6.tar.gz", hash = "sha256:8720fd3104a679853b63eee5b5bd5229fd279105f5d9d08d00dc2de7e7a86700"}, ] [package.dependencies] -"ga4gh.vrs" = ">=2.1.3,<3.0" +"ga4gh.vrs" = ">=2.4.0-a6,<2.5.0" pydantic = ">=2.0,<3.0" [package.extras] @@ -1640,20 +1714,21 @@ tests = ["pytest", "pytest-cov", "pyyaml"] [[package]] name = "ga4gh-va-spec" -version = "0.4.3" +version = "0.5.0a6" description = "GA4GH Variant Annotation (VA) reference implementation" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ga4gh_va_spec-0.4.3-py3-none-any.whl", hash = "sha256:f54330b59fe1a3d27b901d15edbf9f1e3e66e16438a16654f8ef9ac0271876fa"}, - {file = "ga4gh_va_spec-0.4.3.tar.gz", hash = "sha256:a8813745d261e218932608a7905571354e14180be10c2972ed5d03f8b9a7cf72"}, + {file = "ga4gh_va_spec-0.5.0a6-py3-none-any.whl", hash = "sha256:0136e47c6f7ede65daf8981608cf190fc69f3ce94d5c7c1e30c30c86d7bbe97e"}, + {file = "ga4gh_va_spec-0.5.0a6.tar.gz", hash = "sha256:640b4a11a69a87bc80adb1c82c15a9cb2acfd5bf1bf95f9404d3329127e80cef"}, ] [package.dependencies] -"ga4gh.cat_vrs" = ">=0.7.1,<0.8.0" -"ga4gh.vrs" = ">=2.2.0,<3.0" -pydantic = ">=2.0,<3.0" +"ga4gh.cat_vrs" = ">=0.8.0-a6,<0.9.0" +"ga4gh.vrs" = ">=2.4.0-a6,<2.5.0" +pydantic = ">=2.12,<3.0" +typing_extensions = "*" [package.extras] dev = ["pre-commit", "ruff (==0.4.9)"] @@ -1662,14 +1737,14 @@ tests = ["pytest", "pytest-cov", "pyyaml"] [[package]] name = "ga4gh-vrs" -version = "2.3.1" +version = "2.4.0a6" description = "GA4GH Variation Representation Specification (VRS) reference implementation" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ga4gh_vrs-2.3.1-py3-none-any.whl", hash = "sha256:0e0eb6436eb6b229a313eac49e58c36057c20656f3d1c8b3da14a3aaafebcf61"}, - {file = "ga4gh_vrs-2.3.1.tar.gz", hash = "sha256:911cff7eadc8a7ca9e746147d85a90fe1affbb78d68c3f9aca177bdfa9bb07db"}, + {file = "ga4gh_vrs-2.4.0a6-py3-none-any.whl", hash = "sha256:ce6850338ecea7819a5aa49504d0bfe15a1211694e199f20a09139c14f490080"}, + {file = "ga4gh_vrs-2.4.0a6.tar.gz", hash = "sha256:be68882acb659ef25af8b154440c54619e9a921993bae82feb336fd78d2588c3"}, ] [package.dependencies] @@ -2893,6 +2968,38 @@ files = [ {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + +[[package]] +name = "opentelemetry-api" +version = "1.45.0" +description = "OpenTelemetry Python API" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_api-1.45.0-py3-none-any.whl", hash = "sha256:80e068aba7cd56c8b58512d6a36f8d25cb1dfaa0c0a4cc1c938ccf9f362d9cb3"}, + {file = "opentelemetry_api-1.45.0.tar.gz", hash = "sha256:711ede81773c8025c2c03dac0450bc89f3d30aea6eabcc815c570d4e35a963f7"}, +] + +[package.dependencies] +typing-extensions = ">=4.5.0" + [[package]] name = "orcid" version = "1.0.3" @@ -3342,7 +3449,7 @@ description = "C parser in Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "platform_python_implementation != \"PyPy\" and extra == \"server\" and implementation_name != \"PyPy\"" +markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and extra == \"server\"" files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, @@ -3350,20 +3457,21 @@ files = [ [[package]] name = "pydantic" -version = "2.10.6" +version = "2.12.5" description = "Data validation using Python type hints" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pydantic-2.10.6-py3-none-any.whl", hash = "sha256:427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584"}, - {file = "pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236"}, + {file = "pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}, + {file = "pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.27.2" -typing-extensions = ">=4.12.2" +pydantic-core = "2.41.5" +typing-extensions = ">=4.14.1" +typing-inspection = ">=0.4.2" [package.extras] email = ["email-validator (>=2.0.0)"] @@ -3371,116 +3479,137 @@ timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows [[package]] name = "pydantic-core" -version = "2.27.2" +version = "2.41.5" description = "Core functionality for Pydantic validation and serialization" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"}, - {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"}, - {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"}, - {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"}, - {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"}, - {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"}, - {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"}, - {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"}, - {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"}, - {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"}, - {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"}, - {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"}, - {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"}, - {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"}, - {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"}, - {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"}, - {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"}, - {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"}, - {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"}, - {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"}, - {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"}, - {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"}, - {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"}, - {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"}, - {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"}, - {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"}, - {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"}, - {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"}, - {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"}, - {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"}, - {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"}, - {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"}, - {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"}, - {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"}, - {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"}, - {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"}, - {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"}, - {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"}, - {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"}, - {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"}, - {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"}, - {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"}, - {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"}, - {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"}, - {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"}, - {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"}, - {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"}, - {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"}, - {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"}, - {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"}, - {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"}, - {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"}, - {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"}, - {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"}, + {file = "pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146"}, + {file = "pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49"}, + {file = "pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba"}, + {file = "pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9"}, + {file = "pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6"}, + {file = "pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f"}, + {file = "pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7"}, + {file = "pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3"}, + {file = "pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9"}, + {file = "pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd"}, + {file = "pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a"}, + {file = "pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008"}, + {file = "pydantic_core-2.41.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf"}, + {file = "pydantic_core-2.41.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3"}, + {file = "pydantic_core-2.41.5-cp39-cp39-win32.whl", hash = "sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460"}, + {file = "pydantic_core-2.41.5-cp39-cp39-win_amd64.whl", hash = "sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51"}, + {file = "pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e"}, ] [package.dependencies] -typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +typing-extensions = ">=4.14.1" [[package]] name = "pygments" @@ -3693,23 +3822,18 @@ six = ">=1.5" [[package]] name = "python-discovery" -version = "1.3.0" +version = "1.6.1" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "python_discovery-1.3.0-py3-none-any.whl", hash = "sha256:441d9ced3dfce36e113beb35ca302c71c7ef06f3c0f9c227a0b9bb3bd49b9e9f"}, - {file = "python_discovery-1.3.0.tar.gz", hash = "sha256:d098f1e86be5d45fe4d14bf1029294aabbd332f4321179dec85e76cddce834b0"}, + {file = "python_discovery-1.6.1-py3-none-any.whl", hash = "sha256:d43fcdef879fe795352bd13ccf8d185ba5a9f86f36cfcd00529f596e737442b3"}, + {file = "python_discovery-1.6.1.tar.gz", hash = "sha256:cf87d3627dfb4412437fdd5b13eae402607722998d21567993aedbc59b23c15e"}, ] [package.dependencies] filelock = ">=3.15.4" -platformdirs = ">=4.3.6,<5" - -[package.extras] -docs = ["furo (>=2025.12.19)", "sphinx (>=9.1)", "sphinx-autodoc-typehints (>=3.6.3)", "sphinxcontrib-mermaid (>=2)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.5.4)", "pytest (>=8.3.5)", "pytest-mock (>=3.14)", "setuptools (>=75.1)"] [[package]] name = "python-dotenv" @@ -4093,30 +4217,30 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.15.12" +version = "0.6.8" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.15.12-py3-none-linux_armv6l.whl", hash = "sha256:f86f176e188e94d6bdbc09f09bfd9dc729059ad93d0e7390b5a73efe19f8861c"}, - {file = "ruff-0.15.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3bcd123364c3770b8e1b7baaf343cc99a35f197c5c6e8af79015c666c423a6c"}, - {file = "ruff-0.15.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fe87510d000220aa1ed530d4448a7c696a0cae1213e5ec30e5874287b66557b5"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84a1630093121375a3e2a95b4a6dc7b59e2b4ee76216e32d81aae550a832d002"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb129f40f114f089ebe0ca56c0d251cf2061b17651d464bb6478dc01e69f11f5"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0c862b172d695db7598426b8af465e7e9ac00a3ea2a3630ee67eb82e366aaa6"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2849ea9f3484c3aca43a82f484210370319e7170df4dfe4843395ddf6c57bc33"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e77c7e51c07fe396826d5969a5b846d9cd4c402535835fb6e21ce8b28fef847"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b2f4f2f3b1026b5fb449b467d9264bf22067b600f7b6f41fc5958909f449d0"}, - {file = "ruff-0.15.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9ba3b8f1afd7e2e43d8943e55f249e13f9682fde09711644a6e7290eb4f3e339"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e852ba9fdc890655e1d78f2df1499efbe0e54126bd405362154a75e2bde159c5"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dd8aed930da53780d22fc70bdf84452c843cf64f8cb4eb38984319c24c5cd5fd"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01da3988d225628b709493d7dc67c3b9b12c0210016b08690ef9bd27970b262b"}, - {file = "ruff-0.15.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9cae0f92bd5700d1213188b31cd3bdd2b315361296d10b96b8e2337d3d11f53e"}, - {file = "ruff-0.15.12-py3-none-win32.whl", hash = "sha256:d0185894e038d7043ba8fd6aee7499ece6462dc0ea9f1e260c7451807c714c20"}, - {file = "ruff-0.15.12-py3-none-win_amd64.whl", hash = "sha256:c87a162d61ab3adca47c03f7f717c68672edec7d1b5499e652331780fe74950d"}, - {file = "ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f"}, - {file = "ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6"}, + {file = "ruff-0.6.8-py3-none-linux_armv6l.whl", hash = "sha256:77944bca110ff0a43b768f05a529fecd0706aac7bcce36d7f1eeb4cbfca5f0f2"}, + {file = "ruff-0.6.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27b87e1801e786cd6ede4ada3faa5e254ce774de835e6723fd94551464c56b8c"}, + {file = "ruff-0.6.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cd48f945da2a6334f1793d7f701725a76ba93bf3d73c36f6b21fb04d5338dcf5"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:677e03c00f37c66cea033274295a983c7c546edea5043d0c798833adf4cf4c6f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f1476236b3eacfacfc0f66aa9e6cd39f2a624cb73ea99189556015f27c0bdeb"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f5a2f17c7d32991169195d52a04c95b256378bbf0de8cb98478351eb70d526f"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5fd0d4b7b1457c49e435ee1e437900ced9b35cb8dc5178921dfb7d98d65a08d0"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8034b19b993e9601f2ddf2c517451e17a6ab5cdb1c13fdff50c1442a7171d87"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cfb227b932ba8ef6e56c9f875d987973cd5e35bc5d05f5abf045af78ad8e098"}, + {file = "ruff-0.6.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef0411eccfc3909269fed47c61ffebdcb84a04504bafa6b6df9b85c27e813b0"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:007dee844738c3d2e6c24ab5bc7d43c99ba3e1943bd2d95d598582e9c1b27750"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ce60058d3cdd8490e5e5471ef086b3f1e90ab872b548814e35930e21d848c9ce"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1085c455d1b3fdb8021ad534379c60353b81ba079712bce7a900e834859182fa"}, + {file = "ruff-0.6.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:70edf6a93b19481affd287d696d9e311388d808671bc209fb8907b46a8c3af44"}, + {file = "ruff-0.6.8-py3-none-win32.whl", hash = "sha256:792213f7be25316f9b46b854df80a77e0da87ec66691e8f012f887b4a671ab5a"}, + {file = "ruff-0.6.8-py3-none-win_amd64.whl", hash = "sha256:ec0517dc0f37cad14a5319ba7bba6e7e339d03fbf967a6d69b0907d61be7a263"}, + {file = "ruff-0.6.8-py3-none-win_arm64.whl", hash = "sha256:8d3bb2e3fbb9875172119021a13eed38849e762499e3cfde9588e4b4d70968dc"}, + {file = "ruff-0.6.8.tar.gz", hash = "sha256:a5bf44b1aa0adaf6d9d20f86162b34f7c593bfedabc51239953e446aefc8ce18"}, ] [[package]] @@ -4438,20 +4562,20 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlparse" -version = "0.5.5" +version = "0.6.0" description = "A non-validating SQL parser." optional = true -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba"}, - {file = "sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e"}, + {file = "sqlparse-0.6.0-py3-none-any.whl", hash = "sha256:b861c0288ce2fa56209a9a6412d2e066ac664b3873b89c26c9d8415e8e32996f"}, + {file = "sqlparse-0.6.0.tar.gz", hash = "sha256:113c35c75365ab9cc9c7231d68c6428fb11c085fc8e9eb1ad659b7ddbf6cd2b9"}, ] [package.extras] dev = ["build"] -doc = ["sphinx"] +doc = ["furo", "sphinx"] [[package]] name = "stack-data" @@ -4476,23 +4600,23 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] [[package]] name = "starlette" -version = "0.49.3" +version = "1.7.0" description = "The little ASGI library that shines." optional = true -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "starlette-0.49.3-py3-none-any.whl", hash = "sha256:b579b99715fdc2980cf88c8ec96d3bf1ce16f5a8051a7c2b84ef9b1cdecaea2f"}, - {file = "starlette-0.49.3.tar.gz", hash = "sha256:1c14546f299b5901a1ea0e34410575bc33bbd741377a10484a54445588d00284"}, + {file = "starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e"}, + {file = "starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d"}, ] [package.dependencies] -anyio = ">=3.6.2,<5" +anyio = ">=4.0.0,<5" typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\""} [package.extras] -full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] +full = ["httpx (>=0.27.0,<0.29.0)", "httpx2 (>=2.0.0)", "itsdangerous", "jinja2", "opentelemetry-api", "python-multipart (>=0.0.18)", "pyyaml"] [[package]] name = "starlette-context" @@ -4663,16 +4787,31 @@ files = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, - {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, + {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, + {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, ] +[[package]] +name = "typing-inspection" +version = "0.4.4" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147"}, + {file = "typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47"}, +] + +[package.dependencies] +typing-extensions = ">=4.15.0" + [[package]] name = "tzdata" version = "2026.2" @@ -4688,14 +4827,14 @@ markers = {dev = "sys_platform == \"win32\""} [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, + {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"}, + {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"}, ] [package.extras] @@ -4796,23 +4935,83 @@ dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"] docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx_rtd_theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=25.3.0,<25.4.0)", "pycodestyle (>=2.11.0,<2.12.0)"] +[[package]] +name = "variant-annotation" +version = "1.2.0-beta.1" +description = "Genetic variant mapping and annotation pipeline" +optional = true +python-versions = ">=3.11" +groups = ["main"] +markers = "extra == \"server\"" +files = [] +develop = false + +[package.dependencies] +biopython = "*" +boto3 = "*" +click = "*" +openpyxl = "*" +pandas = ">=2.2.1,<3.0.0" +python-dotenv = "*" +redis = "*" +requests = "*" +variant-translation = "1.2.0" + +[package.extras] +dev = ["ruff"] +gnomad = ["hail"] +liftover = ["pyliftover"] +mapping = ["dcd-mapping @ git+https://github.com/VariantEffect/dcd_mapping2.git@v2026.1.2"] +publish = ["build", "twine"] +tests = ["pytest"] + +[package.source] +type = "git" +url = "https://github.com/bbi-lab/variant-annotation.git" +reference = "v1.2.0-beta.1" +resolved_reference = "350b53bae5e99d30ecf6f33821c3df70f5c76a65" + +[[package]] +name = "variant-translation" +version = "1.2.0" +description = "Reverse-translate protein HGVS variants." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "variant_translation-1.2.0-py3-none-any.whl", hash = "sha256:143f242ebcff7b1a2c07bc01283c032f73535c50ddaaaa65e0803a26592f8250"}, + {file = "variant_translation-1.2.0.tar.gz", hash = "sha256:e12a1d7d8ab78ba89999e72fc9203f071b6f799417e575bbf5753e8ca69356b4"}, +] + +[package.dependencies] +click = "*" +hgvs = "*" +python-dotenv = "*" +requests = "*" + +[package.extras] +dev = ["pytest", "ruff"] +publish = ["build", "twine"] + [[package]] name = "virtualenv" -version = "21.3.1" +version = "21.14.1" description = "Virtual Python Environment builder" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "virtualenv-21.3.1-py3-none-any.whl", hash = "sha256:d1a71cf58f2f9228fff23a1f6ec15d39785c6b32e03658d104974247145edd35"}, - {file = "virtualenv-21.3.1.tar.gz", hash = "sha256:c2305bc1fddeec40699b8370d13f8d431b0701f00ce895061ce493aeded4426b"}, + {file = "virtualenv-21.14.1-py3-none-any.whl", hash = "sha256:6fd04089fc0dc33549e7abdff70fc3b63d4e15799f2dbf3281f80d13b9fce522"}, + {file = "virtualenv-21.14.1.tar.gz", hash = "sha256:719b189804e66678017d9f63bbfc590c44f6b96ab4829513806394f9b375929c"}, ] [package.dependencies] distlib = ">=0.3.7,<1" -filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""} +filelock = {version = ">=3.24.2,<5", markers = "python_version >= \"3.10\""} +packaging = ">=23.1" platformdirs = ">=3.9.1,<5" -python-discovery = ">=1.2.2" +python-discovery = ">=1.6" [[package]] name = "watchfiles" @@ -5098,9 +5297,9 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "variant-annotation", "watchtower"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "c32a7c46b7d3c9b818a58a6fa7b3ffc5e28d099969b1e3394b8baedf45585d3d" +content-hash = "5a24b728c64c490a22d95d754e2ffaa6bf7615d01de828803a69f3bfb5e82030" diff --git a/pyproject.toml b/pyproject.toml index 21690de12..126b5606d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "mavedb" -version = "2026.2.7.4" +version = "2026.3.0" description = "API for MaveDB, the database of Multiplexed Assays of Variant Effect." license = "AGPL-3.0-only" readme = "README.md" @@ -34,11 +34,13 @@ email_validator = "~2.1.1" numpy = "~1.26" httpx = "~0.26.0" pandas = ">=2.2.0,<3.0.0" -pydantic = "~2.10.0" +pydantic = "~2.12.0" python-dotenv = "^1.2" python-json-logger = "~2.0.7" SQLAlchemy = "~2.0.29" -ga4gh-va-spec = "~0.4.2" +ga4gh-vrs = { version = "2.4.0a6", allow-prereleases = true } +ga4gh-cat-vrs = { version = "0.8.0a6", allow-prereleases = true } +ga4gh-va-spec = { version = "0.5.0a6", allow-prereleases = true } setuptools = "~81.0.0" # Optional dependencies for running this application as a server @@ -49,9 +51,9 @@ arq = { version = "~0.25.0", optional = true } authlib = { version = "~1.6.6", optional = true } boto3 = { version = "~1.34.97", optional = true } biocommons = { version = "~0.0.0", optional = true } -cryptography = { version = "~46.0.5", optional = true } +cryptography = { version = "~50.0.2", optional = true } cdot = { version = "~0.2.21", optional = true } -fastapi = { version = "~0.121.0", optional = true } +fastapi = { version = "~0.142.2", optional = true } hgvs = { version = "~1.5.4", optional = true } orcid = { version = "~1.0.3", optional = true } pyathena = { version = "~3.14.1", optional = true } @@ -59,10 +61,11 @@ psycopg2 = { version = "~2.9.3", optional = true } python-jose = { extras = ["cryptography"], version = "~3.5.0", optional = true } python-multipart = { version = "~0.0.22", optional = true } requests = { version = "^2.33", optional = true } -starlette = { version = "~0.49.0", optional = true } +starlette = { version = "^1.7.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } +variant-annotation = { git = "https://github.com/bbi-lab/variant-annotation.git", tag = "v1.2.0-beta.1", optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" @@ -87,13 +90,13 @@ types-python-jose = "~3.3.4" types-PyYAML = "~6.0.12.20240808" redis = "~5.0.2" requests-mock = "~1.11.0" -ruff = "~0.15.0" +ruff = "0.6.8" SQLAlchemy = { extras = ["mypy"], version = "~2.0.0" } [tool.poetry.extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "variant-annotation", "watchtower"] [tool.mypy] @@ -104,6 +107,19 @@ plugins = [ ] mypy_path = "mypy_stubs" +# variant-annotation is a git-pinned dependency (not yet published to PyPI) that ships +# no py.typed marker. Treat it as untyped rather than maintaining drift-prone local +# stubs for an actively-developed internal library. +[[tool.mypy.overrides]] +module = "variant_annotation.*" +ignore_missing_imports = true + +# The GA4GH packages ship no py.typed marker, but their pydantic models are fully annotated. Read them +# directly rather than through local stubs, which fall out of step with each release. +[[tool.mypy.overrides]] +module = "ga4gh.*" +follow_untyped_imports = true + [tool.pytest.ini_options] addopts = "-v --import-mode=importlib" asyncio_mode = 'strict' @@ -111,8 +127,8 @@ testpaths = "tests/" pythonpath = "." norecursedirs = "tests/helpers/" markers = """ - integration: tests that exercise multi-component flows and/or database interactions. - unit: fast unit tests with isolated dependencies. + integration: tests that exercise a multi-component flow end-to-end, with no internal mocking. + unit: fast, isolated tests. Real un-mocked collaborators are fine if they're local, fast, and deterministic. network: tests which interact with external services over a network connection. slow: tests which are slow running. """ diff --git a/settings/.env.template b/settings/.env.template index 585bd354f..4574c6dd4 100644 --- a/settings/.env.template +++ b/settings/.env.template @@ -105,7 +105,7 @@ GNOMAD_DATA_VERSION=v4.1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test -S3_ENDPOINT_URL=http://localstack:4566 +S3_ENDPOINT_URL=http://moto:5000 UPLOAD_S3_BUCKET_NAME=score-set-csv-uploads-dev #################################################################################################### diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index b37965a21..4d74f711e 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.7.4" +__version__ = "2026.3.0" logger.info(f"MaveDB {__version__}") diff --git a/src/mavedb/constants.py b/src/mavedb/constants.py index 41f9d825b..0932f858a 100644 --- a/src/mavedb/constants.py +++ b/src/mavedb/constants.py @@ -2,3 +2,5 @@ MAVEDB_BASE_GIT = "https://github.com/VariantEffect/mavedb-api" MAVEDB_FRONTEND_URL = os.getenv("MAVE_FRONTEND_URL", "https://mavedb.org") +# Concept DOI of the public data archive on Zenodo; it always resolves to the latest release. +MAVEDB_BULK_DOWNLOAD_URL = "https://doi.org/10.5281/zenodo.11201736" diff --git a/src/mavedb/data_providers/services.py b/src/mavedb/data_providers/services.py index a94c16d6e..25d85eb2b 100644 --- a/src/mavedb/data_providers/services.py +++ b/src/mavedb/data_providers/services.py @@ -2,7 +2,10 @@ from typing import TYPE_CHECKING, Optional import boto3 +import requests +from biocommons.seqrepo import SeqRepo from cdot.hgvs.dataproviders import ChainedSeqFetcher, FastaSeqFetcher, RESTDataProvider, SeqFetcher +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from mavedb.lib.mapping import VRSMap @@ -16,15 +19,52 @@ DCD_MAP_URL = os.environ.get("DCD_MAPPING_URL", "http://dcd-mapping:8000") CDOT_URL = os.environ.get("CDOT_URL", "http://cdot-rest:8000") +SEQREPO_DIR = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") CSV_UPLOAD_S3_BUCKET_NAME = os.getenv("UPLOAD_S3_BUCKET_NAME", "score-set-csv-uploads-dev") +# (connect, read) seconds for each cdot-rest request. +CDOT_REQUEST_TIMEOUT = (10, 60) + + +class StrictRESTDataProvider(RESTDataProvider): + """cdot's REST data provider, raising on a cdot-rest outage instead of reporting the data missing. + + The base client returns ``None`` for any non-OK response, which reads as "no such transcript" (or "no + transcripts in this region") and is cached for the provider's lifetime. Callers that overwrite stored + projections with a missing transcript would then erase them during an outage. Only a 404, which + cdot-rest returns for an unknown transcript or gene, is reported as missing here. + """ + + def _get_from_url(self, url: str) -> Optional[dict]: + response = requests.get(url, timeout=CDOT_REQUEST_TIMEOUT) + if response.status_code == 404: + return None + + response.raise_for_status() + return response.json() + def seqfetcher() -> ChainedSeqFetcher: return ChainedSeqFetcher(SeqFetcher(), *[FastaSeqFetcher(file) for file in GENOMIC_FASTA_FILES]) def cdot_rest() -> RESTDataProvider: - return RESTDataProvider(url=CDOT_URL, seqfetcher=seqfetcher()) + return StrictRESTDataProvider(url=CDOT_URL, seqfetcher=seqfetcher()) + + +def seqrepo() -> SeqRepo: + return SeqRepo(SEQREPO_DIR) + + +def seqrepo_data_proxy() -> SeqRepoDataProxy: + """VRS sequence/refget data proxy backed by local SeqRepo. + + Distinct from cdot_rest(): cdot is an hgvs *coordinate* data provider, while + AlleleTranslator needs a VRS *sequence* proxy that can derive_refget_accession. + Mirrors dcd_mapping's SeqRepo-backed translator. Uses the same HGVS_SEQREPO_DIR + location as deps.get_seqrepo and the refget router. + """ + return SeqRepoDataProxy(seqrepo()) def vrs_mapper(url: Optional[str] = None) -> VRSMap: diff --git a/src/mavedb/db/mixins.py b/src/mavedb/db/mixins.py new file mode 100644 index 000000000..c9b17f4b3 --- /dev/null +++ b/src/mavedb/db/mixins.py @@ -0,0 +1,220 @@ +"""Reusable declarative mixins.""" + +from datetime import datetime +from typing import ClassVar, Optional, Sequence, TypeVar + +from sqlalchemy import ColumnElement, DateTime, Update, func, select, update +from sqlalchemy import inspect as sa_inspect +from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property +from sqlalchemy.orm import Mapped, Session, mapped_column + +T = TypeVar("T", bound="ValidTime") + + +class ValidTime: + """Valid-time versioning for rows that change over time (transaction-time SCD Type 2). + + Applies to either a versioned entity (a row that is replaced by a newer version of the same + logical thing — e.g. a mapping record) or a link/association row (a relationship that comes + and goes). Immutable, content-addressed rows (deduplicated entities like an allele or a + source-versioned annotation record) do not use this — their "what applies now" is answered by + the links that reference them, not by versioning the row itself. + + A row is live while ``valid_to`` is NULL. Superseding a row sets its ``valid_to`` to the + successor's ``valid_from`` instead of deleting it, so the full history is retained and a + point-in-time query is a single half-open ``[valid_from, valid_to)`` predicate. ``current`` + and ``as_of`` express that predicate so call sites never hand-roll it. + + Convention: replacing a live row with a successor goes through :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk), which stamp the retired ``valid_to`` and the new + ``valid_from`` with one timestamp so the handoff is gap-free regardless of transaction + boundaries. ``retire`` (single) / ``retire_live_where`` (bulk) are *withdrawal* primitives — + retiring with no successor; do not pair a bare retire with a separate insert, or a slow job + between the two opens a window where the row is neither live-old nor live-new. The method pairs + mirror: ``retire``/``supersede_with`` act on one row, ``*_live_where`` act on a predicate. + + ``current`` is derived (``valid_to IS NULL``), not stored — one source of truth, no + dual-write to keep consistent. Each table using this mixin should add a partial unique index + over its natural key ``WHERE valid_to IS NULL`` to enforce a single live row per key (the link + key for an association, the logical-entity key for an entity). + + Transaction time only: ``valid_from``/``valid_to`` record when *we* held the row, not when an + external source considered it effective. Tables fed by externally-versioned sources (e.g. a + ClinVar release, a gnomAD version) should add their own source-version column for that axis. + + Consumer contract — a model mixing this in, and any code writing it, MUST honor: + + 1. **Add the partial unique index.** Declare a unique index over the natural key + ``WHERE valid_to IS NULL`` (one live row per key). It is the loud backstop: forgetting to + retire a predecessor before inserting its successor raises ``IntegrityError`` instead of + silently leaving two live rows. + 2. **Replace through supersede, never retire-then-insert.** Use :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk) to swap a live row for a successor. A bare + :meth:`retire` followed by a separate insert is the gap footgun — if a slow job or a commit + falls between them, the retired ``valid_to`` and the new ``valid_from`` come from different + transaction clocks and a point-in-time query lands in a hole where the row is neither + old-live nor new-live. Reserve ``retire``/``retire_live_where`` for genuine withdrawal (no + successor). Gaps only arise on a *same-key* handoff; the cascade in (3) only ever retires + children under a superseded parent's old key, so it never needs a supersede. + 3. **Declare ``__retire_cascade__`` for parent rows with ``ValidTime`` children.** A live link to + a retired parent is stale; retiring a parent must retire its live child links. The ORM + relationship cascade does not do this (it fires only on hard DELETE). Bulk + :meth:`supersede_live_where` refuses to run on a class that declares ``__retire_cascade__`` + precisely because it cannot cascade — supersede such rows one at a time with + :meth:`supersede_with`. + 4. **Filter reads with ``current``/``as_of``.** Never read live rows by assuming a query returns + only current state; constrain with ``current`` (or ``as_of(ts)``) explicitly. Because of (3), + a live link implies a live parent, so allele/parent-side ``current`` queries do not surface + links dangling off retired parents. + """ + + valid_from: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + valid_to: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + + # Names of relationships to child ``ValidTime`` rows that should be retired when this row is. + # A link to a retired row is itself stale, so closing a parent's window must close its live + # links' windows too — the relationship cascade ("all, delete-orphan") only fires on a hard + # DELETE, not on this soft ``valid_to`` close. Each named relationship must target a + # ``ValidTime`` model; ``retire`` issues one bulk UPDATE per relationship. + # + # This covers only the transaction-time axis: "the parent was superseded, so its links are too." + # Links retired on a source-version axis (e.g. a new gnomAD release superseding an + # allele→gnomAD-variant link while the allele itself stays live) are a separate trigger and are + # not driven from here. Cascade is one level deep (bulk UPDATE, not per-row ``retire``); a deeper + # chain would need its own handling. + __retire_cascade__: ClassVar[tuple[str, ...]] = () + + @hybrid_property + def current(self) -> bool: + return self.valid_to is None + + @current.inplace.expression + @classmethod + def _current_expression(cls) -> ColumnElement[bool]: + return cls.valid_to.is_(None) + + @classmethod + def as_of(cls, ts: datetime) -> ColumnElement[bool]: + """Filter clause selecting the rows live at ``ts`` (half-open ``[valid_from, valid_to)``).""" + return (cls.valid_from <= ts) & (cls.valid_to.is_(None) | (cls.valid_to > ts)) + + @hybrid_method + def live_at(self, as_of: Optional[datetime]) -> bool: + """Live at ``as_of``, or currently live when ``as_of`` is ``None``. + + The one-call form of the ``as_of(ts) if ts is not None else current`` branch every temporal + read otherwise repeats. Same result as :attr:`current` / :meth:`as_of`, but usable uniformly on + the class, an instance, and an ``aliased()`` class (SQLAlchemy adapts the column refs to it).""" + if as_of is None: + return self.valid_to is None + return self.valid_from <= as_of and (self.valid_to is None or self.valid_to > as_of) + + @live_at.expression + @classmethod + def _live_at_expression(cls, as_of: Optional[datetime]) -> ColumnElement[bool]: + if as_of is None: + return cls.valid_to.is_(None) + return (cls.valid_from <= as_of) & (cls.valid_to.is_(None) | (cls.valid_to > as_of)) + + def retire(self, session: Optional[Session] = None, at: Optional[datetime] = None) -> None: + """Close this row's validity window, cascading to the live child links named in + ``__retire_cascade__``. Idempotent — an already-closed row keeps its original valid_to. + + Pass ``at`` to stamp an explicit ``valid_to`` shared with a successor's ``valid_from``; this + is how :meth:`supersede_with` keeps the handoff gap-free. Without ``at`` the close uses + ``func.now()`` (Postgres ``transaction_timestamp``). This is a *withdrawal* primitive on its + own — to replace a row with a successor, use :meth:`supersede_with` so the handoff is closed. + + ``session`` is required only when ``__retire_cascade__`` is non-empty (the cascade issues a + bulk UPDATE per child relationship). A leaf row with no declared cascades retires without one. + The cascade runs even when this row was already closed, so a half-finished prior retire (row + closed, links not) still converges. + """ + stamp = at if at is not None else func.now() + if self.valid_to is None: + self.valid_to = stamp + + if not self.__retire_cascade__: + return + if session is None: + raise ValueError( + f"{type(self).__name__}.retire() needs a session to cascade-retire {self.__retire_cascade__}." + ) + + mapper = sa_inspect(type(self)) + assert mapper is not None # a mapped class always inspects to a Mapper + for name in self.__retire_cascade__: + rel = mapper.relationships[name] + child_cls = rel.mapper.class_ + conditions = [remote == getattr(self, local.key) for local, remote in rel.local_remote_pairs] + session.execute(child_cls.retire_live_where(*conditions, at=at)) + + @classmethod + def retire_live_where(cls, *conditions: ColumnElement[bool], at: Optional[datetime] = None) -> Update: + """An UPDATE that retires the live rows matching ``conditions`` (closes their valid_to). + Only currently-live rows are touched, so already-retired rows keep their original valid_to. + Pass ``at`` to stamp an explicit ``valid_to`` (used by :meth:`supersede`); defaults to + ``func.now()``. + + This is a *withdrawal* primitive — retiring live rows with no successor. To replace live rows + with new ones, use :meth:`supersede_live_where` so retire and insert share one timestamp. + + Usage: ``session.execute(Model.retire_live_where(Model.foo == bar))``. + """ + return ( + update(cls).where(cls.valid_to.is_(None), *conditions).values(valid_to=at if at is not None else func.now()) + ) + + def supersede_with(self: T, session: Session, replacement: T, at: Optional[datetime] = None) -> T: + """Retire this row (cascading to its child links) and insert ``replacement``, stamping the + retired row's ``valid_to`` and the replacement's ``valid_from`` with one timestamp so the + handoff has no gap — independent of how many transactions or how much wall-clock separate + them. The single-row counterpart to :meth:`supersede_live_where`, for superseding one known + row that may carry ``__retire_cascade__`` children. + + Flushes the retire before the insert: the partial unique index (one live row per natural key) + is checked per statement and the unit of work emits INSERTs before UPDATEs, so without the + flush the replacement and the not-yet-closed original would both be live and trip the index. + """ + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + self.retire(session, at=at) + session.flush() + replacement.valid_from = at + session.add(replacement) + return replacement + + @classmethod + def supersede_live_where( + cls, + session: Session, + new_rows: Sequence[T], + *conditions: ColumnElement[bool], + at: Optional[datetime] = None, + ) -> Sequence[T]: + """Retire the live rows matching ``conditions`` and insert ``new_rows``, stamping both sides + with one timestamp so every retired row's ``valid_to`` equals every new row's ``valid_from`` + — a gap-free handoff independent of transaction boundaries. The bulk counterpart to + :meth:`supersede_with` (and the supersede form of :meth:`retire_live_where`), for replacing a + *set* of live rows in one scope (e.g. a score set's derived links) with a freshly computed + set. + + The retire executes immediately (before the inserts flush), so a reused natural key does not + trip the partial unique index. Only for leaf ``ValidTime`` rows: a bulk retire cannot fire + ``__retire_cascade__``, so this refuses to run on a class that declares one — use + :meth:`supersede_with` per row there. + """ + if cls.__retire_cascade__: + raise ValueError( + f"{cls.__name__} declares __retire_cascade__; bulk supersede cannot cascade to children. " + "Use supersede_with per row so child links retire too." + ) + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + session.execute(cls.retire_live_where(*conditions, at=at)) + for row in new_rows: + row.valid_from = at + session.add_all(new_rows) + return new_rows diff --git a/src/mavedb/db/session.py b/src/mavedb/db/session.py index 8c0127ac4..c667d7299 100644 --- a/src/mavedb/db/session.py +++ b/src/mavedb/db/session.py @@ -12,6 +12,8 @@ DB_PASSWORD = os.getenv("DB_PASSWORD") DB_POOL_SIZE = int(os.getenv("DB_POOL_SIZE", "5")) DB_MAX_OVERFLOW = int(os.getenv("DB_MAX_OVERFLOW", "10")) +# Kept short so a request waiting on the pool fails inside the load balancer's idle timeout. +DB_POOL_TIMEOUT = int(os.getenv("DB_POOL_TIMEOUT", "10")) DB_URL = f"postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_DATABASE_NAME}" @@ -19,6 +21,7 @@ DB_URL, pool_size=DB_POOL_SIZE, max_overflow=DB_MAX_OVERFLOW, + pool_timeout=DB_POOL_TIMEOUT, pool_pre_ping=True, ) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/src/mavedb/db/timeouts.py b/src/mavedb/db/timeouts.py new file mode 100644 index 000000000..5ecc1011f --- /dev/null +++ b/src/mavedb/db/timeouts.py @@ -0,0 +1,105 @@ +"""Database timeouts for the API. + +The API sets its limits on each connection it opens, so the worker, scripts and migrations, which share the +same login and legitimately run long statements, are unaffected. Prod connects to RDS directly; behind RDS +Proxy these session-level ``SET``s would pin every connection and belong on the login instead. +""" + +import logging +import os +import time + +from sqlalchemy import event, text +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session + +from mavedb.lib.logging.context import increment_logging_context, logging_context + +logger = logging.getLogger(__name__) + +# Each limit is in seconds and can be overridden from the environment; 0 disables it, as in Postgres. +# The prod API load balancer's idle timeout is 60s. A statement's limit plus the 10s pool wait stays under it. +API_STATEMENT_TIMEOUT_SECONDS = int(os.getenv("DB_STATEMENT_TIMEOUT_SECONDS", "30")) +CSV_STATEMENT_TIMEOUT_SECONDS = int(os.getenv("DB_CSV_STATEMENT_TIMEOUT_SECONDS", "45")) +# Below the statement timeout, so API writes queued behind a worker's bulk write fail fast. +API_LOCK_TIMEOUT_SECONDS = int(os.getenv("DB_LOCK_TIMEOUT_SECONDS", "5")) +# Above the longest legitimate gap between statements inside one request's transaction. +API_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS = int(os.getenv("DB_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS", "60")) +# Not a limit: statements slower than this are logged so the limits above can be tuned from real data. +SLOW_STATEMENT_SECONDS = float(os.getenv("DB_SLOW_STATEMENT_SECONDS", "5")) + +_STARTED_AT_ATTR = "_mavedb_statement_started_at" +_SLOW_STATEMENT_LOG_LENGTH = 200 + + +def _set_api_timeouts(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + try: + cursor.execute(f"SET statement_timeout = '{API_STATEMENT_TIMEOUT_SECONDS}s'") + cursor.execute(f"SET lock_timeout = '{API_LOCK_TIMEOUT_SECONDS}s'") + cursor.execute(f"SET idle_in_transaction_session_timeout = '{API_IDLE_IN_TRANSACTION_TIMEOUT_SECONDS}s'") + finally: + cursor.close() + + # psycopg2 opens a transaction for the first statement; commit so the SETs are not rolled back. + dbapi_connection.commit() + + +def apply_api_timeouts(engine: Engine) -> None: + """Apply the API's timeouts to every new connection of ``engine``. Safe to call repeatedly. + + Call this before the first connection is opened: connections already in the pool are not touched. + """ + if not event.contains(engine, "connect", _set_api_timeouts): + event.listen(engine, "connect", _set_api_timeouts) + + +def allow_long_statements(db: Session, seconds: int = CSV_STATEMENT_TIMEOUT_SECONDS) -> None: + """Raise ``statement_timeout`` for the rest of the current transaction. + + Transaction-scoped, so it does not outlive the request. A ``commit()`` or ``rollback()`` ends it, so call + this immediately before the long read. Does nothing while the API's statement timeout is disabled, so + disabling it disables it everywhere. + """ + if not API_STATEMENT_TIMEOUT_SECONDS: + return + + db.execute(text("SELECT set_config('statement_timeout', :ms, true)"), {"ms": str(seconds * 1000)}) + + +def observe_statements(engine: Engine, threshold_seconds: float) -> None: + """Count each request's statements on ``engine``, and log any statement slower than ``threshold_seconds``. + + The count lands in the request's canonical log as ``db_statement_count``, so a route whose query count + grows with its data shows up on real data, where test fixtures are too small to show it. + + A slow statement cancelled by ``statement_timeout`` is logged too, since those are the ones worth tuning. + Logs the duration and the start of the SQL text, never the bound parameters, which can hold user data. + """ + + def _log(statement: str, started_at: float) -> None: + elapsed = time.perf_counter() - started_at + if elapsed < threshold_seconds: + return + + text_start = " ".join(statement.split())[:_SLOW_STATEMENT_LOG_LENGTH] + logger.warning( + msg=f"Slow database statement ({elapsed:.1f}s): {text_start}", + extra={**logging_context(), "statement_seconds": round(elapsed, 3)}, + ) + + @event.listens_for(engine, "before_cursor_execute") + def _start_timer(_conn, _cursor, _statement, _parameters, context, _executemany): + increment_logging_context("db_statement_count") + setattr(context, _STARTED_AT_ATTR, time.perf_counter()) + + @event.listens_for(engine, "after_cursor_execute") + def _log_completed(_conn, _cursor, statement, _parameters, context, _executemany): + _log(statement, getattr(context, _STARTED_AT_ATTR)) + + @event.listens_for(engine, "handle_error") + def _log_failed(exception_context): + context = exception_context.execution_context + started_at = getattr(context, _STARTED_AT_ATTR, None) + if started_at is not None and exception_context.statement: + _log(exception_context.statement, started_at) diff --git a/src/mavedb/deps.py b/src/mavedb/deps.py index 8e4ce5106..712a6e5a3 100644 --- a/src/mavedb/deps.py +++ b/src/mavedb/deps.py @@ -1,4 +1,3 @@ -import os from typing import Any, AsyncGenerator, Generator from arq import ArqRedis, create_pool @@ -6,7 +5,7 @@ from cdot.hgvs.dataproviders import RESTDataProvider from sqlalchemy.orm import Session -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo from mavedb.db.session import SessionLocal from mavedb.worker.settings import RedisWorkerSettings @@ -33,5 +32,4 @@ def hgvs_data_provider() -> RESTDataProvider: def get_seqrepo() -> SeqRepo: - seqrepo_dir = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") - return SeqRepo(seqrepo_dir) + return seqrepo() diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py new file mode 100644 index 000000000..1cd1fcf9e --- /dev/null +++ b/src/mavedb/lib/allele_annotations.py @@ -0,0 +1,222 @@ +"""Digest-keyed annotation assembly over the deduplicated allele model. + +Gathers a set of alleles' external annotations — VEP consequence, gnomAD frequency, ClinVar +assertions — keyed by ``vrs_digest``. The digest is the join key the serving envelopes use to ride +annotations *alongside* the spec-pure Cat-VRS members: Cat-VRS itself carries no annotation slot, so +annotations travel beside it in a flat digest-keyed map. + +All three sources are ``ValidTime`` and ``allele_id``-keyed with no reverse collection on +``Allele`` (they are navigated set-wise from the link tables). ``as_of`` reconstructs them at a past +instant via ``live_at``, defaulting to the currently-live rows. Absence is normal and encoded as +``None``/empty — annotations are sparse and computed at all levels with no fixed level→source +mapping (D44/D44b): a genomic allele *may* carry VEP, a coding allele *may* match gnomAD. + +VEP and gnomAD are one-live-per-allele, so each is at most one value; ClinVar is multi-live (one +live link per release), so it is a list. + +This is the shared assembler for both ``GET /variants/{urn}`` (all of a variant's alleles) and +``GET /alleles/{digest}`` (a single allele) — pass whichever allele set the caller has in hand. +:func:`get_allele_cross_references` is the identifiers-only counterpart for callers that need only +the gnomAD and ClinVar identifiers (the Cat-VRS ``mappings``), skipping VEP and the payload columns. +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, Sequence + +from sqlalchemy import Result, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class VepAnnotation: + """A VEP molecular consequence for an allele and the Ensembl release it was resolved under.""" + + consequence: Optional[str] + source_version: str + + +@dataclass(frozen=True) +class GnomadAnnotation: + """gnomAD population frequency for an allele, from its live gnomAD link.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] + db_version: str + db_identifier: str + + +@dataclass(frozen=True) +class ClinvarAnnotation: + """One ClinVar assertion for an allele (one per live release — an allele may carry several).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] + clinvar_allele_id: str + db_version: str + + +@dataclass(frozen=True) +class GnomadReference: + """An allele's gnomAD variant ID and the gnomAD release it was matched in.""" + + db_identifier: str + db_version: str + + +@dataclass +class AlleleCrossReferences: + """An allele's identifiers in external variant registries. Identifiers only; no annotation payload.""" + + gnomad: Optional[GnomadReference] = None + clinvar_variation_ids: list[str] = field(default_factory=list) + + +@dataclass +class AlleleAnnotations: + """The external annotations for a single allele. Every field is sparse — a source with no data + for the allele is ``None`` (vep/gnomad) or an empty list (clinvar).""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = field(default_factory=list) + + def cross_references(self) -> AlleleCrossReferences: + """The identifiers these annotations carry, for a caller that already fetched the full map.""" + return AlleleCrossReferences( + gnomad=( + GnomadReference(db_identifier=self.gnomad.db_identifier, db_version=self.gnomad.db_version) + if self.gnomad is not None + else None + ), + clinvar_variation_ids=[c.clinvar_variation_id for c in self.clinvar if c.clinvar_variation_id], + ) + + +def get_allele_annotations( + db: Session, alleles: Sequence[Allele], *, as_of: Optional[datetime] = None +) -> dict[str, AlleleAnnotations]: + """Assemble the digest-keyed annotation map for ``alleles``. + + Returns one :class:`AlleleAnnotations` per distinct ``vrs_digest`` present in ``alleles``. An + allele with no annotations still gets an (empty) entry, so the map's keys mirror the alleles + handed in — the envelope can join every Cat-VRS member to a (possibly empty) annotation block. + ``as_of`` reconstructs each source at a past instant; it defaults to the currently-live rows. + """ + # id -> digest for the alleles we were handed; digest is the map key the envelope joins on. + digest_by_id = {allele.id: allele.vrs_digest for allele in alleles} + annotations: dict[str, AlleleAnnotations] = { + digest: AlleleAnnotations() for digest in digest_by_id.values() if digest is not None + } + if not digest_by_id: + return annotations + + allele_ids = list(digest_by_id.keys()) + + # VEP — at most one live consequence per allele. + vep_rows = db.execute( + select( + VepAlleleConsequence.allele_id, + VepAlleleConsequence.functional_consequence, + VepAlleleConsequence.source_version, + ) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.live_at(as_of)) + ).tuples() + for allele_id, consequence, source_version in vep_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].vep = VepAnnotation(consequence=consequence, source_version=source_version) + + # gnomAD — at most one live link per allele, resolving to a frequency record. + gnomad_rows: Result[tuple[int, GnomADVariant]] = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + ) + for allele_id, gnomad_variant in gnomad_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].gnomad = GnomadAnnotation( + allele_frequency=gnomad_variant.allele_frequency, + allele_count=gnomad_variant.allele_count, + allele_number=gnomad_variant.allele_number, + faf95_max=gnomad_variant.faf95_max, + db_version=gnomad_variant.db_version, + db_identifier=gnomad_variant.db_identifier, + ) + + # ClinVar — multi-live (one live link per release), so each allele accumulates a list. + clinvar_rows: Result[tuple[int, ClinvarControl]] = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + ) + for allele_id, clinvar_control in clinvar_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].clinvar.append( + ClinvarAnnotation( + clinical_significance=clinvar_control.clinical_significance, + clinical_review_status=clinvar_control.clinical_review_status, + clinvar_variation_id=clinvar_control.clinvar_variation_id, + clinvar_allele_id=clinvar_control.db_identifier, + db_version=clinvar_control.db_version, + ) + ) + + return annotations + + +def get_allele_cross_references( + db: Session, alleles: Sequence[Allele], *, as_of: Optional[datetime] = None +) -> dict[str, AlleleCrossReferences]: + """Assemble the digest-keyed gnomAD and ClinVar identifiers for ``alleles``. + + Keyed and time-scoped like :func:`get_allele_annotations`, but reads only the identifier columns: + two queries, no VEP. An allele with no identifiers still gets an (empty) entry. + """ + digest_by_id = {allele.id: allele.vrs_digest for allele in alleles} + references: dict[str, AlleleCrossReferences] = { + digest: AlleleCrossReferences() for digest in digest_by_id.values() if digest is not None + } + if not digest_by_id: + return references + + allele_ids = list(digest_by_id.keys()) + + gnomad_rows = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant.db_identifier, GnomADVariant.db_version) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + ).tuples() + for allele_id, db_identifier, db_version in gnomad_rows: + digest = digest_by_id[allele_id] + if digest is not None: + references[digest].gnomad = GnomadReference(db_identifier=db_identifier, db_version=db_version) + + clinvar_rows = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl.clinvar_variation_id) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + ).tuples() + for allele_id, variation_id in clinvar_rows: + digest = digest_by_id[allele_id] + if digest is not None and variation_id is not None: + references[digest].clinvar_variation_ids.append(variation_id) + + return references diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py new file mode 100644 index 000000000..b372970e2 --- /dev/null +++ b/src/mavedb/lib/allele_detail.py @@ -0,0 +1,192 @@ +"""The allele-detail view backing ``GET /alleles/{digest|CAID}``. + +The allele-grain counterpart of :mod:`lib.variant_detail`. Where the variant view anchors on a +*measurement*, this anchors on an *allele* (a VRS digest, or a CAID naming the nt-canonical change) +and serves that allele's identity, its full cross-layer **equivalence class**, and the digest-keyed +annotations that ride alongside. + +It is measurement-agnostic, and that shapes what it does *not* carry: + +- No score, counts, classification, or version standing. Those are properties of a measured variant, + not of an allele (an allele is deduplicated by digest and shared across every score set mapping to it). +- No spec-pure Cat-VRS. The ``CategoricalVariant`` / ``DefiningAlleleConstraint`` is emitted only by the + measurement view, rooted at the measured allele. Here the members are labelled on a flat map + *relative to the focus allele*. + +**The equivalence class is the full cross-record union** (:func:`lib.alleles.get_allele_translations`): +every allele co-linked to any live record touching the focus, restricted by the caller to records of +score sets they may read. That is the discoverable "everything related to this change" set. Each member +is then labelled relative to the focus allele. Even though this level is unable to determine which +allele was actually measured, we can still say how a given allele relates to the focus allele (e.g., +faithful, candidate, convergent) based on the graph structure. + +``as_of`` reconstructs the molecular layer, class membership + annotations, at the past instant. The +focus allele's own identity is immutable and content-addressed. ``as_of`` defaults to this instant. +""" + +from collections.abc import Collection +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleIdentity +from mavedb.lib.alleles import get_allele_translations +from mavedb.lib.cat_vrs import member_label +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.mapping_record_allele import MappingRecordAllele + + +@dataclass(frozen=True) +class AlleleDetail: + """The assembled allele-detail envelope (transit; serialized by ``view_models.allele_detail``).""" + + # Own identity (the focus allele itself). + digest: str + level: Optional[str] + """The sequence level of the allele (e.g., nucleotide or amino acid).""" + hgvs: Optional[str] + """The reference-frame HGVS string (exactly one of hgvs_g/c/p, depending on level).""" + clingen_allele_id: Optional[str] + vrs: Optional[dict[str, Any]] + + # Cross-layer equivalence class (all alleles equivalent to the focus). + alleles: dict[str, AlleleIdentity] + """The full cross-layer equivalence class, keyed by VRS digest and sharing keys with `annotations`. + Each member is an `AlleleIdentity` labelled relative to the focus. The focus allele(s) carry + `is_focus=True`.""" + + # External annotations. + annotations: dict[str, AlleleAnnotations] + """External annotations for each member of the cross-layer equivalence class, keyed by VRS digest.""" + + +def _focus_projection_pairs( + db: Session, + focus_alleles: list[Allele], + *, + as_of: Optional[datetime], + score_set_ids: Optional[Collection[int]], +) -> dict[str, str]: + """A ``digest -> digest`` map pairing each focus allele with its projection. + + A ``projection_group`` is one variant at each level, local to a single mapping record. A nucleotide + focus pairs with the ≤1 *other* nucleotide allele in its group (c↔g, symmetric). A protein focus pairs + with nothing: it is what its group's nucleotide alleles encode, not another spelling of them. Reading any + one live link is safe: the focus holds one live link per record it appears in, and each record re-encodes the same + change onto the same content-addressed digests, so the pairing is record-independent (assuming those + records agree on genome assembly). + + This is the one place the projection-vs-convergent-encoding distinction needs the link-level + ``projection_group`` — every other member is classified by level alone. + """ + pairs: dict[str, str] = {} + for focus in focus_alleles: + if focus.vrs_digest is None or focus.level not in NUCLEOTIDE_LEVELS: + continue + + # limit(1) rather than one_or_none(), which would raise for any allele measured more than once. + link_query = ( + select(MappingRecordAllele) + .where(MappingRecordAllele.allele_id == focus.id) + .where(MappingRecordAllele.projection_group.isnot(None)) + .where(MappingRecordAllele.live_at(as_of)) + ) + if score_set_ids is not None: + link_query = link_query.where(MappingRecordAllele.score_set_id.in_(score_set_ids)) + + link = db.scalar(link_query.limit(1)) + if link is None: + continue + + partners = ( + select(Allele.vrs_digest) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(MappingRecordAllele.mapping_record_id == link.mapping_record_id) + .where(MappingRecordAllele.projection_group == link.projection_group) + .where(MappingRecordAllele.allele_id != focus.id) + .where(Allele.level.in_(NUCLEOTIDE_LEVELS)) + .where(MappingRecordAllele.live_at(as_of)) + ) + for projection in db.scalars(partners).all(): + if projection is not None: + pairs[focus.vrs_digest] = projection + pairs.setdefault(projection, focus.vrs_digest) + + return pairs + + +def get_allele_detail( + db: Session, + focus: Allele, + *, + focus_digests: set[str], + as_of: Optional[datetime] = None, + score_set_ids: Optional[Collection[int]] = None, +) -> AlleleDetail: + """Assemble the allele-detail envelope anchored on ``focus``. + + ``focus_digests`` is the set of VRS digests that are the *focus* of this view. A single digest for + a by-digest fetch, or the (genomic + coding) representations of a CAID for a by-CAID fetch. The + equivalence class is every allele co-linked to a live record touching the ``focus`` + (:func:`get_allele_translations`). An orphan allele falls back to ``focus`` alone. Non-focus members + are labelled by :func:`~mavedb.lib.cat_vrs.member_label`. ``score_set_ids``, when given, restricts the equivalence + class and projection pairing to records of those score sets. + """ + equivalence_class = get_allele_translations(db, focus.id, as_of=as_of, score_set_ids=score_set_ids) or [focus] + annotations = get_allele_annotations(db, equivalence_class, as_of=as_of) + + focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [focus] + # A PAID focuses the protein allele; a CAID or digest focuses nucleotide allele(s). + focus_level = ( + SequenceLevel.protein.value + if any(a.level == SequenceLevel.protein.value for a in focus_alleles) + else focus_alleles[0].level + ) + + pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of, score_set_ids=score_set_ids) + # The focus's projection(s) that are not themselves focus (for a CAID fetch the projection pair is + # jointly the focus, so this is empty and they are simply flagged is_focus instead). + projection_digests = {pairs[d] for d in focus_digests if d in pairs} - focus_digests + + alleles: dict[str, AlleleIdentity] = {} + for member in equivalence_class: + digest = member.vrs_digest + if digest is None: + continue + + # Determine the relation and derivation for non-focus members. Focus + # members have no relation or derivation relative to themselves. + relation, derivation = None, None + if digest not in focus_digests: + rel, der = member_label(focus_level, member.level, is_projection=digest in projection_digests) + relation = rel.value if rel is not None else None + derivation = der + + alleles[digest] = AlleleIdentity( + level=member.level, + hgvs=member.hgvs_g or member.hgvs_c or member.hgvs_p, + clingen_allele_id=member.clingen_allele_id, + is_focus=digest in focus_digests, + relation=relation, + derivation=derivation, + projection_of=pairs.get(digest), + ) + + return AlleleDetail( + digest=focus.vrs_digest or "", + level=focus.level, + hgvs=focus.hgvs_g or focus.hgvs_c or focus.hgvs_p, + clingen_allele_id=focus.clingen_allele_id, + vrs=vrs_object_from_mapped_variant(focus.post_mapped).model_dump(mode="json", exclude_none=True) + if focus.post_mapped is not None + else None, + alleles=alleles, + annotations=annotations, + ) diff --git a/src/mavedb/lib/allele_identity.py b/src/mavedb/lib/allele_identity.py new file mode 100644 index 000000000..6ae589cf9 --- /dev/null +++ b/src/mavedb/lib/allele_identity.py @@ -0,0 +1,96 @@ +"""The MaveDB molecular-identity view of an allele — shared by variant detail and allele detail. + +Both serving views present a set of alleles keyed by VRS digest, each labelled by how it relates to +the allele the view is anchored on (its *focus*). The two views differ only in what the focus is: +``GET /variants/{urn}`` focuses the **measured** allele, ``GET /alleles/{digest}`` focuses the +**queried** allele. The identity shape is shared and its labels are always read *relative to the +focus*. + +This module owns that shared shape (:class:`AlleleIdentity`) and its provenance vocabulary +(:class:`AlleleDerivation`). The Cat-VRS structural relation codes live in :mod:`lib.cat_vrs` +(``CatVrsRelation``); this module carries the derivation (confidence/provenance) axis. + +Why ``relation`` and ``derivation`` are separate axes +---------------------------------------------------- +The two axes describe the *same* pair of alleles — member and focus — but ask questions from different +domains. ``relation`` is **biological**: how does this allele relate to the focus? ``derivation`` is +**provenance**: how did we arrive at this allele from the focus, and hence how much should it be +trusted? Neither answer determines the other, because a biological relationship does not say how it was +found and a route does not say what it found. + +Half of that independence is already visible in the tree: ``projection`` covers *both* ``translation_of`` +(nt→protein) and ``coordinate_representation_of`` (nt↔nt) — one route, two different biological +relationships. The other half is latent only because today exactly one pipeline path produces each +relationship. A ``co_encodes`` allele is biologically "a different codon encoding the same protein +change" however we reached it; reverse translation from the focus, a registry walk, and an independent +measurement in another score set are three different provenance claims about the same biology. + +Two consequences that make the split load-bearing rather than tidy: + +1. **Cat-VRS cannot express confidence, by design.** ``encodes`` is a claim about molecules: this + nucleotide allele encodes that protein change. It is equally true of a codon we measured and one + we guessed. A structural representation spec has no vocabulary for "we do not know which of these + six codons was in the well". +2. **``candidate`` is a fact about our data, not about the molecule.** A protein-level assay reported + an amino-acid change; which codon produced it is unknown *to MaveDB*, not unknown biologically. + Attempting to encode that on a molecular-relationship axis would misstate what the molecule does. + +Today's one-to-one mapping between the two vocabularies is **a fact about today's pipeline, not about +these axes. Never infer one from the other.** They are emitted together by ``cat_vrs.member_label``, the +single source of truth for every view. +""" + +from dataclasses import dataclass +from enum import Enum +from typing import Optional + + +class AlleleDerivation(str, Enum): + """How an allele's representation was arrived at, *relative to the focus allele*: the + confidence/provenance axis, and the one the UI badges on. + + There is deliberately **no** ``authoritative`` value: the focus allele is marked by + :attr:`AlleleIdentity.is_focus`, not by a derivation. That keeps the axis meaningful even when a + variant was not explicitly measured. See the module docstring for why this axis is separate from + the Cat-VRS ``relation``, and why neither may be inferred from the other. + """ + + PROJECTION = "projection" + """Deterministic and precise, given (assembly, transcript): the focus's projection + (nucleotide↔nucleotide) and its protein consequence (nucleotide→protein).""" + + CANDIDATE = "candidate" + """Reverse-translation of a *protein* focus: genuinely ambiguous. One member of the fanned-out + equivalence class, not a precise and deterministic pair.""" + + CONVERGENT = "convergent" + """A distinct, precisely-known nucleotide change that *converges* on the same protein consequence + a different codon produces. Not ambiguous like a candidate nor a projection *of* the focus. A + separate, unmeasured variant that simply shares the consequence.""" + + +@dataclass(frozen=True) +class AlleleIdentity: + """The MaveDB molecular-identity facts for one allele in a view's ``alleles`` map, keyed by VRS + digest. + + These four axes all read *relative to the view's focus allele*: + + - ``is_focus``: whether this allele is the one the view is anchored on. Its ``relation``/``derivation`` + are ``None``. This is the reference point the others are described against. + - ``relation`` (Cat-VRS, biological): how this allele relates to the focus in a biological sense. + - ``derivation`` (:class:`AlleleDerivation`, provenance): How we arrived at this allele from the focus, + hence how much to trust it. + - ``projection_of`` (provenance): the VRS digest of this allele's c↔g projection, when + known. ``None`` for the protein apex, pre-reverse-translation data, or where the pairing is not + resolved in this view. + """ + + is_focus: bool + level: Optional[str] + hgvs: Optional[str] + clingen_allele_id: Optional[str] + + relation: Optional[str] + derivation: Optional[AlleleDerivation] + projection_of: Optional[str] diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py new file mode 100644 index 000000000..587188a25 --- /dev/null +++ b/src/mavedb/lib/allele_measurements.py @@ -0,0 +1,406 @@ +"""The variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. + +Given a ClinGen allele id, return every measurement related to it across sequence levels. Two measurements +are related when their mapping records share an allele. A ``CA`` anchors on its nt alleles; a ``PA`` on the +protein allele. Each measurement is labelled by its own measured level relative to the query: ``direct`` +(measured at the query), ``protein_consequence``, or ``nucleotide_encoding``. + +Example. Reference codon ``TTT`` (Phe200): both ``c.598T>C`` and the encoding ``c.600T>A`` encode +``p.Phe200Leu``. Call them ``CA1``, ``CA2``, ``PA1``. Score set X assays ``c.598T>C`` (nt), score set Y +assays ``p.Phe200Leu`` (protein), score set Z assays ``c.600T>A`` (nt): + + query CA1 → X direct, Y protein_consequence, Z nucleotide_encoding + query PA1 → Y direct, X and Z nucleotide_encoding + +A CA query reaches its protein consequence (Y) through the shared protein node ``PA1``. This is what makes +Y reachable even when a protein assay has not been reverse-translated: its record then links only the +protein node, with no nt allele for the CA anchor to match, so the apex is the only path to it. The +encoding Z is reached without the apex: reverse translation links every record to its full synonymous nt set, +so Z's record already links ``CA1`` directly. + +The protein-consequence step is resolved once, in :func:`_resolve_protein_apex`, which also measures it: +a consequence resolving to more than one protein id (a data-integrity signal) is logged, and the number of +consequences reached with no reverse-translation data of their own is written to the request log. + +Authorization (``has_permission``, per ``lib/score_sets.py``): score-set READ gates whether a measurement +appears; calibration READ gates only its inline classification. ``as_of`` reconstructs which records and +links were live at a past instant. + +Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the record-scoped detail of one +measurement. +""" + +import logging +from dataclasses import dataclass +from datetime import date, datetime +from typing import Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, aliased, joinedload + +from mavedb.lib.alleles import readable_score_set_ids_by_allele +from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_calibrations import ( + calibration_preference_key, + classification_evidence_strength, + classifications_by_variant, +) +from mavedb.lib.types.authentication import UserData +from mavedb.lib.variants import variant_score +from mavedb.models.allele import Allele +from mavedb.models.enums.measurement_relationship import MeasurementRelationship +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class AlleleMeasurement: + """One measurement in the query's equivalence class. + + ``assay_level`` is the sequence level at which this measurement was assayed. ``classification`` is the + readable call for this variant. It comes from a non-RUO calibration unless the caller opted into research-use-only + calls and the score set has no readable non-RUO calibration with functional classifications; then it may come + from an RUO calibration, and ``classification_is_research_use_only`` says so. A non-RUO calibration that leaves + this variant unclassified is a call in itself and is not overridden. + """ + + variant_urn: str + score: Optional[float] + assay_level: Optional[SequenceLevel] + relationship: str + assay_level_hgvs: Optional[str] + submitted_hgvs: Optional[str] + score_set_urn: str + score_set_title: str + classification: Optional[ScoreCalibrationFunctionalClassification] + classification_is_research_use_only: bool + is_current: bool + superseded_by_score_set: Optional[str] + + +@dataclass(frozen=True) +class _ProteinApex: + """Protein consequence(s) co-membered with a CA anchor's nt alleles — the shared node a CA query + reaches protein measurements through. One PAID is well-formed; ``len(caids) > 1`` is a data-quality + signal (a duplicate or divergent consequence). ``unregistered`` counts apex rows without a PAID yet. + + nt→protein is deterministic only per (assembly, transcript), and the anchor includes the CAID's + transcript-agnostic genomic allele. So records targeting different transcripts of one gene reach + different consequences legitimately, and >1 PAID is a signal to investigate rather than proof of + corruption. + """ + + allele_ids: list[int] + caids: list[str] + unregistered: int + + +def _preferred_classification( + classifications: list[tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]], + *, + user_data: Optional[UserData], + research_use_only: bool = False, +) -> Optional[ScoreCalibrationFunctionalClassification]: + """The variant's preferred *readable* functional classification among ``classifications`` (its + classification/calibration pairs, from :func:`classifications_by_variant`), or ``None``. + + Mirrors the UI's calibration cascade — ``primary`` then ``investigator_provided``, strongest evidence + within a tier, then ``id`` for determinism. Calibrations the caller can't read are skipped rather than + blanking a readable call. + + Draws from one side of the RUO line only: non-RUO calibrations by default (the call MaveDB asserts, matching + the annotation layer's default), or RUO calibrations alone when ``research_use_only`` is set, for display. + """ + candidates = [ + (classification, calibration) + for classification, calibration in classifications + if calibration.research_use_only is research_use_only + and has_permission(user_data, calibration, Action.READ).permitted + ] + if not candidates: + return None + + def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]) -> tuple: + classification, calibration = pair + magnitude, _ = classification_evidence_strength(classification) + return ( + *calibration_preference_key(calibration), + -magnitude, + calibration.id, + ) + + return min(candidates, key=preference)[0] + + +def _has_readable_clinical_calibration(score_set: ScoreSet, *, user_data: Optional[UserData]) -> bool: + """Whether the score set has any non-RUO calibration the caller can read, classifying this variant or not. + + A calibration without functional classifications (one that only records controls or a baseline score) can't + classify any variant, so its silence is no answer and it doesn't count. + """ + return any( + not calibration.research_use_only + and calibration.functional_classifications + and has_permission(user_data, calibration, Action.READ).permitted + for calibration in score_set.score_calibrations + ) + + +# Experiment keywords shown on the variant page's measurement table; how many are filled in breaks evidence ties. +ASSAY_FACT_KEYWORD_KEYS = frozenset( + { + "Phenotypic Assay Method", + "Molecular Mechanism Assessed", + "Phenotypic Assay Model System", + } +) + + +def _assay_fact_count(score_set: ScoreSet) -> int: + """How many of :data:`ASSAY_FACT_KEYWORD_KEYS` the score set's experiment provides.""" + keyword_objs = score_set.experiment.keyword_objs if score_set.experiment is not None else [] + return len({obj.controlled_keyword.key for obj in keyword_objs} & ASSAY_FACT_KEYWORD_KEYS) + + +def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date], assay_fact_count: int) -> tuple: + """Sort: current before superseded; direct before related; strongest evidence (pathogenic wins ties) + first; most assay facts provided first; newest-published first; then URN for a stable tiebreak. + + Only non-RUO calls rank, so opting into research-use-only calls never reorders the list. + """ + clinical = None if measurement.classification_is_research_use_only else measurement.classification + magnitude, direction = classification_evidence_strength(clinical) + return ( + 0 if measurement.is_current else 1, + 0 if measurement.relationship == MeasurementRelationship.direct else 1, + 0 if clinical is not None else 1, + -magnitude, + direction, + -assay_fact_count, + -(published_date.toordinal()) if published_date is not None else 0, + measurement.variant_urn, + ) + + +def _resolve_protein_apex( + db: Session, anchor_ids: list[int], *, as_of: Optional[datetime], score_set_ids: set[int] +) -> _ProteinApex: + """The protein consequence(s) co-membered with the nt ``anchor_ids`` via a shared live record. + + The single site the apex is resolved, so its cardinality (see :class:`_ProteinApex`) is measured + rather than being a silent side effect of the union. Reads ``clingen_allele_id`` to count PAIDs. + + Only records of ``score_set_ids`` are followed. A protein consequence that only an unreadable score + set's mapping produced would otherwise pull in public measurements of that protein change, disclosing + the private one. + """ + anchor_link = aliased(MappingRecordAllele) + rows = db.execute( + select(Allele.id, Allele.clingen_allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) + .where(anchor_link.allele_id.in_(anchor_ids)) + .where(anchor_link.score_set_id.in_(score_set_ids)) + .where(anchor_link.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(Allele.level == SequenceLevel.protein.value) + .distinct() + ).all() + + return _ProteinApex( + allele_ids=[row.id for row in rows], + caids=sorted({row.clingen_allele_id for row in rows if row.clingen_allele_id is not None}), + unregistered=sum(1 for row in rows if row.clingen_allele_id is None), + ) + + +def get_allele_measurements( + db: Session, + clingen_allele_id: str, + *, + user_data: Optional[UserData], + include_superseded: bool = False, + include_research_use_only: bool = False, + as_of: Optional[datetime] = None, +) -> list[AlleleMeasurement]: + """The measurements in ``clingen_allele_id``'s equivalence class, or ``[]`` if it resolves to no live + record. ``include_research_use_only`` lets a research-use-only call stand in where the score set has no + readable clinical calibration (see :class:`AlleleMeasurement`). + + A CA query returns ``direct`` measurements, its ``protein_consequence``, and the ``nucleotide_encoding`` + members. A PA query returns its ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. + """ + anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() + + # An allele only the caller's unreadable score sets link is treated as absent, as on GET /alleles. + readable_by_allele = readable_score_set_ids_by_allele(db, user_data, [row.id for row in anchor]) + anchor = [row for row in anchor if row.id in readable_by_allele] + if not anchor: + return [] + + readable_score_set_ids: set[int] = set().union(*readable_by_allele.values()) + + anchor_ids = [row.id for row in anchor] + entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) + + # A CA query always includes its protein consequence, reached through the shared protein node, so it is + # reachable even when a protein assay isn't reverse-translated yet. This is calculated transitively, but + # the calculation should be deterministic. + apex: Optional[_ProteinApex] = None + if not entry_is_protein: + apex = _resolve_protein_apex(db, anchor_ids, as_of=as_of, score_set_ids=readable_score_set_ids) + anchor_ids += apex.allele_ids + + # Warn if multiple distinct protein apexes were resolved for this CA. + if len(apex.caids) > 1: + logger.warning( + msg=( + f"ClinGen allele {clingen_allele_id} resolved to {len(apex.caids)} distinct protein " + f"apexes ({', '.join(apex.caids)}); measurements may conflate consequences." + ), + extra=logging_context(), + ) + + record_ids = db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id.in_(anchor_ids)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + if not record_ids: + return [] + + # Records that carry a derived (RT) link of their own. A protein consequence whose record has none was + # reached only through the apex of an already resolved protein consequence. + records_with_derived_links: set[int] = ( + set( + db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(MappingRecordAllele.is_authoritative.is_(False)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + ) + if apex is not None + else set() + ) + + # Each record's authoritative (measured) allele fixes the assayed level and the direct/related call. + authoritative = aliased(MappingRecordAllele) + rows = ( + db.execute( + select(MappingRecord, Variant, Allele) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join( + authoritative, + and_( + authoritative.mapping_record_id == MappingRecord.id, + authoritative.is_authoritative.is_(True), + authoritative.live_at(as_of), + ), + ) + .join(Allele, Allele.id == authoritative.allele_id) + .where(MappingRecord.id.in_(record_ids)) + .where(MappingRecord.live_at(as_of)) + .options( + joinedload(Variant.score_set) + .joinedload(ScoreSet.experiment) + .selectinload(Experiment.keyword_objs) + .joinedload(ExperimentControlledKeywordAssociation.controlled_keyword), + # Read per measurement below; loaded once per score set rather than lazily per row. + joinedload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + joinedload(Variant.score_set).selectinload(ScoreSet.superseding_score_set), + ) + ) + .tuples() + .all() + ) + + classifications = classifications_by_variant(db, [variant.id for _, variant, _ in rows]) + + measurements: list[tuple[AlleleMeasurement, Optional[date], int]] = [] + clinically_calibrated: dict[int, bool] = {} + protein_consequence_count = 0 + apex_only_unresolved_count = 0 + for record, variant, measured_allele in rows: + score_set = variant.score_set + if not has_permission(user_data, score_set, Action.READ).permitted: + continue + + # Don't leak an unreadable superseding score set. + superseding = score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + is_current = superseding is None + if not is_current and not include_superseded: + continue + + # Label by the measured (authoritative) allele's level relative to the query. + if measured_allele.clingen_allele_id == clingen_allele_id: + relationship = MeasurementRelationship.direct + elif measured_allele.level == SequenceLevel.protein.value: + relationship = MeasurementRelationship.protein_consequence + else: + relationship = MeasurementRelationship.nucleotide_encoding + + # Apex-only provenance: a protein consequence whose record has no derived link was reached purely + # through the apex. + if apex is not None and relationship == MeasurementRelationship.protein_consequence: + protein_consequence_count += 1 + if record.id not in records_with_derived_links: + apex_only_unresolved_count += 1 + + assay_level = SequenceLevel(record.assay_level) if record.assay_level else None + variant_classifications = classifications.get(variant.id, []) + classification = _preferred_classification(variant_classifications, user_data=user_data) + classification_is_research_use_only = False + if include_research_use_only and classification is None: + if score_set.id not in clinically_calibrated: + clinically_calibrated[score_set.id] = _has_readable_clinical_calibration(score_set, user_data=user_data) + if not clinically_calibrated[score_set.id]: + classification = _preferred_classification( + variant_classifications, user_data=user_data, research_use_only=True + ) + classification_is_research_use_only = classification is not None + + measurement = AlleleMeasurement( + variant_urn=variant.urn or "", + score=variant_score(variant), + assay_level=assay_level, + relationship=relationship, + assay_level_hgvs=record.hgvs_assay_level, + submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, + score_set_urn=score_set.urn or "", + score_set_title=score_set.title or "", + classification=classification, + classification_is_research_use_only=classification_is_research_use_only, + is_current=is_current, + superseded_by_score_set=superseding.urn if superseding is not None else None, + ) + measurements.append((measurement, score_set.published_date, _assay_fact_count(score_set))) + + # Publish the apex provenance to the request log so the ambiguous-apex and apex-only-unresolved rates are observable. + if apex is not None: + save_to_logging_context( + { + "apex_paid_count": len(apex.caids), + "apex_unregistered_count": apex.unregistered, + "measurements_protein_consequence": protein_consequence_count, + "measurements_apex_only_unresolved": apex_only_unresolved_count, + } + ) + + measurements.sort(key=lambda entry: _ordering_key(*entry)) + return [measurement for measurement, _, _ in measurements] diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py new file mode 100644 index 000000000..01a208043 --- /dev/null +++ b/src/mavedb/lib/alleles.py @@ -0,0 +1,234 @@ +"""Allele-graph queries over the deduplicated allele model. + +These traverse the ``MappingRecordAllele`` link graph rather than any external identifier, because an +allele's cross-layer equivalence (its genomic / coding / protein representations) is established by the +mapping + reverse-translation process and materialized as co-membership in a ``MappingRecord``'s allele +set. No single identifier spans the layers: ClinGen's canonical allele id (CAID) covers only the +nucleotide layers, the protein allele carries a distinct PA, so the link graph is the only thing that +ties all three together. +""" + +from collections.abc import Callable, Collection, Iterator, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Optional, TypeVar + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, joinedload + +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.types.authentication import UserData +from mavedb.lib.utils import batched +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +# How many variants a whole-set builder loads per round of batched queries. Bounds both the size of each +# query's IN list (a variant's record links can fan out to dozens of reverse-translation members) and the +# number of ORM rows held at once. +VARIANT_CHUNK_SIZE = 500 + +VariantT = TypeVar("VariantT") +InputsT = TypeVar("InputsT") + + +@dataclass(frozen=True) +class LiveRecordLinks: + """A variant's live (or as-of) ``MappingRecord``, that record's live ``MappingRecordAllele`` links (each with + its ``allele`` and projection ``group`` eagerly loaded).""" + + record: MappingRecord + links: list[MappingRecordAllele] + + +def readable_score_set_ids_by_allele( + db: Session, user_data: Optional[UserData], allele_ids: Collection[int] +) -> dict[int, set[int]]: + """Map each of ``allele_ids`` to the ids of the score sets linking it that the caller may read. + + Every allele exists because some score set's mapping produced it, so these are the score sets through + which the caller may see it. An allele with no readable score set is omitted. Links are considered + regardless of valid time: a retired link still records why the allele was created. + """ + if not allele_ids: + return {} + + links = db.execute( + select(MappingRecordAllele.allele_id, MappingRecordAllele.score_set_id) + .where(MappingRecordAllele.allele_id.in_(allele_ids)) + .distinct() + ).all() + score_sets = db.scalars(select(ScoreSet).where(ScoreSet.id.in_({score_set_id for _, score_set_id in links}))).all() + readable = {score_set.id for score_set in score_sets if has_permission(user_data, score_set, Action.READ).permitted} + + by_allele: dict[int, set[int]] = {} + for allele_id, score_set_id in links: + if score_set_id in readable: + by_allele.setdefault(allele_id, set()).add(score_set_id) + + return by_allele + + +def with_chunk_inputs( + variants: Sequence[VariantT], load: Callable[[Sequence[VariantT]], InputsT] +) -> Iterator[tuple[VariantT, InputsT]]: + """Pair each variant with the inputs ``load`` builds for its chunk, one :data:`VARIANT_CHUNK_SIZE` chunk at + a time, so a whole-set builder runs a fixed number of queries per chunk rather than per variant.""" + for chunk in batched(variants, VARIANT_CHUNK_SIZE): + inputs = load(chunk) + for variant in chunk: + yield variant, inputs + + +def get_live_record_allele_links( + db: Session, variant_ids: Collection[int], *, as_of: Optional[datetime] = None +) -> dict[int, LiveRecordLinks]: + """Return each variant's live ``MappingRecord`` and its live allele links, keyed by variant id. + + Two queries whatever the number of variants, so whole-set builders load a chunk of variants at a time + rather than one variant per round trip. A variant with no live record is absent from the result. + + This is **record-scoped**, deliberately unlike :func:`get_allele_translations`: it stays within each + variant's own mapping record rather than taking the cross-record union an anchor allele can belong to. + That scope is what the per-variant Cat-VRS transit needs — the variant's measured (authoritative) + allele as the defining representation and exactly its co-linked members, not the equivalence class + assembled from every record that happens to share a deduplicated allele. + + Temporal: defaults to the currently-live records and links (``valid_to IS NULL``). ``as_of`` applies + the same half-open predicate to both, so each set is evaluated at one instant. + """ + if not variant_ids: + return {} + + records = db.scalars( + select(MappingRecord).where(MappingRecord.variant_id.in_(variant_ids)).where(MappingRecord.live_at(as_of)) + ).all() + if not records: + return {} + + links_by_record: dict[int, list[MappingRecordAllele]] = {record.id: [] for record in records} + for link in db.scalars( + select(MappingRecordAllele) + .where(MappingRecordAllele.mapping_record_id.in_(links_by_record)) + .where(MappingRecordAllele.live_at(as_of)) + .options(joinedload(MappingRecordAllele.allele), joinedload(MappingRecordAllele.group)) + ): + links_by_record[link.mapping_record_id].append(link) + + return {record.variant_id: LiveRecordLinks(record, links_by_record[record.id]) for record in records} + + +def find_variants_by_vrs_identifier( + db: Session, identifier: str, *, as_of: Optional[datetime] = None +) -> list[tuple[Variant, Optional[Allele]]]: + """Resolve a GA4GH VRS identifier to the variants it identifies, before or after mapping. + + Matches ``identifier`` against both sides of a mapping: + + * a deduplicated allele's ``vrs_digest`` (the reference-level, post-mapped variation), walking + ``Allele → MappingRecordAllele → MappingRecord → Variant``. Alleles are shared across variants and + score sets, so one identifier can resolve to several variants; each is paired with the matched allele. + * a mapping record's ``vrs_digest`` (the pre-mapped variation on the submitted target sequence). Each + such variant is paired with its measured (authoritative) allele, or ``None`` when it did not map. + + The live link + record set (``valid_to IS NULL`` / ``as_of``) is used to filter results. Returns distinct + ``(variant, allele)`` pairs ordered by score set, then variant number, then allele. Include an ``as_of`` + timestamp to reconstruct the set as it stood at a past instant. + """ + post_mapped = ( + select(Variant, Allele) + .select_from(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Allele.vrs_digest == identifier) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + # Callers check read permission on each match's score set; load them here rather than one per match. + .options(joinedload(Variant.score_set)) + ) + pre_mapped = ( + select(Variant, Allele) + .select_from(MappingRecord) + .join(Variant, Variant.id == MappingRecord.variant_id) + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative, + MappingRecordAllele.live_at(as_of), + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(MappingRecord.vrs_digest == identifier) + .where(MappingRecord.live_at(as_of)) + .options(joinedload(Variant.score_set)) + ) + + # An accession-based target's pre- and post-mapped variations can be the same, so both queries may + # return a pair. + matches: dict[tuple[int, Optional[int]], tuple[Variant, Optional[Allele]]] = {} + for stmt in (post_mapped, pre_mapped): + for variant, allele in db.execute(stmt).unique().tuples(): + matches.setdefault((variant.id, allele.id if allele is not None else None), (variant, allele)) + return sorted( + matches.values(), + key=lambda match: ( + match[0].score_set_id, + match[0].variant_number or 0, + match[0].id, + match[1].id if match[1] is not None else 0, + ), + ) + + +def get_allele_translations( + db: Session, + allele_id: int, + *, + as_of: Optional[datetime] = None, + score_set_ids: Optional[Collection[int]] = None, +) -> list[Allele]: + """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a + ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, + and protein layers. + + The relation is co-membership in a record's allele set, not a shared identifier — see the module + docstring. Because alleles are deduplicated by ``vrs_digest`` and shared across variants/score sets, + the anchor may belong to several ``MappingRecord``s; the result is the union of their allele sets + (normally the same biological equivalence class). Scope by record or score set upstream if a single + context is required. + + Temporal: by default returns the currently-live set (``valid_to IS NULL``). Pass ``as_of`` to + reconstruct the set as it stood at a past instant — both the anchor hop and the fan-out hop apply + the same half-open ``[valid_from, valid_to)`` predicate, so the whole set is evaluated at one + instant. The retire-cascade invariant (a live link implies a live record) holds under ``as_of`` too, + so filtering the links alone is sufficient. + + ``score_set_ids``, when given, restricts the union to records of those score sets (e.g. the ones the + caller may read). Only the anchor hop needs the filter: every link of a record shares its score set. + """ + record_query = ( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id == allele_id) + .where(MappingRecordAllele.live_at(as_of)) + ) + if score_set_ids is not None: + record_query = record_query.where(MappingRecordAllele.score_set_id.in_(score_set_ids)) + + record_ids = db.scalars(record_query).all() + if not record_ids: + return [] + + return list( + db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + ) diff --git a/src/mavedb/lib/annotation/annotate.py b/src/mavedb/lib/annotation/annotate.py index 83b13b714..ddf0f40c9 100644 --- a/src/mavedb/lib/annotation/annotate.py +++ b/src/mavedb/lib/annotation/annotate.py @@ -1,11 +1,15 @@ """ -This module supports the construction of three main VA-Spec data structures based on the MaveDB MappedVariant object: +This module supports the construction of three main VA-Spec data structures from a variant's +:class:`VariantAnnotationContext` (the new mapping-record substrate — see ``context.py``): - StudyResult See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#study-result-structure - Statement See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#statement-structure - VariantPathogenicityStatement See: https://va-spec.ga4gh.org/en/latest/va-standard-profiles/community-profiles/acmg-2015-profiles.html#variant-pathogenicity-statement-acmg-2015 + +Callers build the context at the edge (router/script) via ``variant_annotation_context`` and pass it in; +these builders never touch the database. ``as_of`` selection and supersession are baked into the context. """ from typing import Optional, Sequence, TypeVar, Union @@ -13,28 +17,30 @@ from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement +from mavedb.lib.annotation.calibration import ( + calibration_scope_extension, + calibrations_available_for_annotation, + select_strongest_functional_calibration, + select_strongest_pathogenicity_calibration, +) from mavedb.lib.annotation.classification import functional_classification_of_variant +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.annotation.eligibility import ( + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, - mapped_variant_to_pathogenicity_statement, -) -from mavedb.lib.annotation.study_result import mapped_variant_to_experimental_variant_impact_study_result -from mavedb.lib.annotation.util import ( - calibration_scope_extension, - calibrations_available_for_annotation, - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, - select_strongest_functional_calibration, - select_strongest_pathogenicity_calibration, + functional_statement, + pathogenicity_statement, ) +from mavedb.lib.annotation.study_result import variant_impact_study_result from mavedb.lib.permissions.principal import Principal -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration Annotation = TypeVar( @@ -56,90 +62,94 @@ def _disclosing_calibration_scope(annotation: Annotation, calibrations: Sequence ) -def variant_study_result(mapped_variant: MappedVariant) -> ExperimentalVariantFunctionalImpactStudyResult: +def variant_study_result(context: VariantAnnotationContext) -> ExperimentalVariantFunctionalImpactStudyResult: # A study result reports the measured score and carries no calibration-derived evidence, so its scope # is public regardless of viewer. Disclosed anyway, so that a missing scope never has to be read as # "public" or "generated before disclosure existed". - return _disclosing_calibration_scope(mapped_variant_to_experimental_variant_impact_study_result(mapped_variant), []) + return _disclosing_calibration_scope(variant_impact_study_result(context), []) def variant_functional_impact_statement( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False, principal: Optional[Principal] = None, ) -> Optional[Statement]: if not can_annotate_variant_for_functional_statement( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal + context, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) eligible_calibrations = calibrations_available_for_annotation( - mapped_variant, + context, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal, ) # Select the calibration with the strongest evidence - strongest_calibration, strongest_range = select_strongest_functional_calibration( - mapped_variant, eligible_calibrations - ) + strongest_calibration, strongest_range = select_strongest_functional_calibration(context, eligible_calibrations) if not strongest_calibration: return None # Get the classification from the strongest range # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Build evidence lines for all eligible calibrations functional_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence.append(functional_evidence_line(mapped_variant, score_calibration, [study_result])) + functional_evidence.append(functional_evidence_line(context, score_calibration, [study_result])) return _disclosing_calibration_scope( - mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + functional_statement( + context, functional_proposition, functional_evidence, strongest_calibration, classification ), eligible_calibrations, ) def variant_pathogenicity_statement( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False, principal: Optional[Principal] = None, ) -> Optional[VariantPathogenicityStatement]: if not can_annotate_variant_for_pathogenicity_evidence( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal + context, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) eligible_calibrations = calibrations_available_for_annotation( - mapped_variant, + context, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal, ) # Select the calibration with the strongest evidence - strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration( - mapped_variant, eligible_calibrations - ) + strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration(context, eligible_calibrations) if not strongest_calibration: return None + # The statement carries one proposition (targeted by every evidence line), so its disease context + # is the strongest calibration's — the same calibration that anchors the ACMG classification. + # + # TODO#XXX - Pooling calibrations with different disease contexts is not supported by the current VA-Spec model. If a + # variant has multiple calibrations with different disease contexts, the strongest calibration is used for the + # statement's disease context and we should consider whether to filter out calibrations with different disease contexts + # from the evidence lines. + clinical_proposition = variant_pathogenicity_proposition(context, strongest_calibration) + # Get the classification from the strongest range (used for the functional statement within clinical evidence) # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Note: strongest_range is used in the pathogenicity statement for ACMG classification # If None, the statement will use UNCERTAIN_SIGNIFICANCE @@ -147,41 +157,44 @@ def variant_pathogenicity_statement( # Build evidence lines for all eligible calibrations clinical_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, [functional_evidence], score_calibration, classification + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + functional_impact_statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, classification ) clinical_evidence.append( - acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + acmg_evidence_line(context, score_calibration, clinical_proposition, [functional_impact_statement]) ) return _disclosing_calibration_scope( - mapped_variant_to_pathogenicity_statement( - mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + pathogenicity_statement( + context, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range ), eligible_calibrations, ) def variant_highest_level_annotation( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, principal: Optional[Principal] = None, ) -> Optional[Union[ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement]]: """ - Build the single highest-materialized VA-Spec layer for a mapped variant. + Build the single highest-materialized VA-Spec layer for a variant's annotation context. Layer ladder (highest to lowest): pathogenicity statement -> functional impact statement -> study result. - Returns None when the variant has no post-mapped allele and therefore cannot be annotated. + + Returns None when the variant lacks the target/mapping data required to build any layer. The viewer decides which layer is reachable as well as what the layer contains: a variant whose only calibration is invisible to this principal degrades to a study result rather than yielding a statement with nothing in it. """ + try: - if can_annotate_variant_for_pathogenicity_evidence(mapped_variant, principal=principal): - return variant_pathogenicity_statement(mapped_variant, principal=principal) - if can_annotate_variant_for_functional_statement(mapped_variant, principal=principal): - return variant_functional_impact_statement(mapped_variant, principal=principal) - return variant_study_result(mapped_variant) + if can_annotate_variant_for_pathogenicity_evidence(context, principal=principal): + return variant_pathogenicity_statement(context, principal=principal) + if can_annotate_variant_for_functional_statement(context, principal=principal): + return variant_functional_impact_statement(context, principal=principal) + return variant_study_result(context) + except MappingDataDoesntExistException: return None diff --git a/src/mavedb/lib/annotation/calibration.py b/src/mavedb/lib/annotation/calibration.py new file mode 100644 index 000000000..13565a84f --- /dev/null +++ b/src/mavedb/lib/annotation/calibration.py @@ -0,0 +1,259 @@ +"""Score-calibration eligibility and selection for VA-Spec annotation. + +Which of a score set's calibrations may back an annotation (``score_calibration_may_be_used_for_annotation``) +and, among the eligible ones, which carries the strongest evidence for a given variant +(``select_strongest_*``). The selection resolves ties/conflicts to the conservative call (normal / uncertain +significance). Consumed by the annotation builders in ``annotate.py``. +""" + +from typing import Iterable, Literal, Optional + +from ga4gh.core.models import Extension +from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided + +from mavedb.lib.annotation.classification import ( + ExperimentalVariantFunctionalImpactClassification, + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification + +CALIBRATION_SCOPE_EXTENSION_NAME = "mavedb_calibration_scope" +"""Extension naming the principal an annotation was built for. See ``calibration_scope_extension``.""" + + +def calibrations_available_for_annotation( + context: VariantAnnotationContext, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, + principal: Optional[Principal] = None, +) -> list[ScoreCalibration]: + """ + Select the calibrations on a mapped variant's score set that may build the requested annotation. + + Two independent questions decide this, and they are asked by different collaborators. + - Eligibility: Does this calibration carry the classifications the annotation type needs, and is + its research-use-only standing permitted here. This is ``score_calibration_may_be_used_for_annotation``. + - Visibility: May this principal read it at all. This is the viewer's, because a calibration's READ rule + is stricter than its score set's. + + Args: + context (VariantAnnotationContext): The context containing the variant whose score set's calibrations are considered. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to be built. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller, so a function that omits it gets public calibrations only. + + Returns: + list[ScoreCalibration]: The eligible, visible calibrations, in score set order. + """ + viewer = (principal if principal is not None else Principal()).viewer_for(ScoreCalibrationViewer) + + return [ + score_calibration + for score_calibration in viewer.visible(context.variant.score_set.score_calibrations) + if score_calibration_may_be_used_for_annotation( + score_calibration, + annotation_type, + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + ) + ] + + +def calibration_scope_extension(calibrations: Iterable[ScoreCalibration]) -> Extension: + """ + Describe the principal an annotation was built for, given the calibrations behind it. + + VA-Spec statements carry no stable identifier, so two callers can receive materially different + statements from the same URL. Naming the scope on the object itself is what keeps that honest: a + consumer holding a record can tell whether it is the one anyone would get, or one widened by the + requester's own access. + + Args: + calibrations (Iterable[ScoreCalibration]): The calibrations contributing evidence to the annotation. + + Returns: + Extension: A ``mavedb_calibration_scope`` extension, ``restricted`` when any contributing + calibration is private and ``public`` otherwise. + """ + if any(calibration.private for calibration in calibrations): + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="restricted", + description=( + "Built from at least one private score calibration, visible to the requesting viewer. " + "Another viewer requesting this variant may receive fewer evidence lines, or none." + ), + ) + + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="public", + description=( + "Built only from public score calibrations. Any viewer requesting this variant receives the same evidence." + ), + ) + + +def score_calibration_may_be_used_for_annotation( + score_calibration: ScoreCalibration, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, +) -> bool: + """ + Check if a score calibration may be used for annotation based on its properties. + + This function evaluates whether a given score calibration is suitable for use in + annotation by checking its research use only status and the presence of required + classifications based on the annotation type. + + Args: + score_calibration (ScoreCalibration): The score calibration to evaluate. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation + being considered, which determines the required classifications for validity. + allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations + marked as research use only for annotation. Defaults to False. + + Returns: + bool: True if the score calibration can be used for annotation, False otherwise. + """ + if score_calibration.research_use_only and not allow_research_use_only_calibrations: + return False + + if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: + return False + + if annotation_type == "pathogenicity" and all( + fr.acmg_classification is None for fr in score_calibration.functional_classifications + ): + return False + + return True + + +def select_strongest_functional_calibration( + context: VariantAnnotationContext, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for functional classification. + + In case of ties or conflicting classifications, defaults to normal classification. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should be classified as INDETERMINATE but still receive annotations. + """ + if not calibrations: + return None, None + + # Collect all calibrations and their classifications + candidates: list[ + tuple[ + ScoreCalibration, + ScoreCalibrationFunctionalClassification, + ExperimentalVariantFunctionalImpactClassification, + ] + ] = [] + + for calibration in calibrations: + functional_range, classification = functional_classification_of_variant(context.variant, calibration) + if functional_range is not None: + candidates.append((calibration, functional_range, classification)) + + # If variant is not in any range, return first calibration with None to indicate INDETERMINATE + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Check if all classifications agree + classifications = [c[2] for c in candidates] + if all(cls == classifications[0] for cls in classifications): + # All agree, return the first one + return candidates[0][0], candidates[0][1] + + # Conflict exists: default to normal classification + normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] + if normal_candidates: + return normal_candidates[0][0], normal_candidates[0][1] + + # If no normal classification, return the first candidate + return candidates[0][0], candidates[0][1] + + +def select_strongest_pathogenicity_calibration( + context: VariantAnnotationContext, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for pathogenicity classification. + + Uses ACMG evidence strength to determine the strongest evidence. + In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain + significance by returning None for the functional range. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should receive annotations even though it's not classified in any range. + """ + if not calibrations: + return None, None + + # Define evidence strength ordering (higher index = stronger evidence) + # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. + # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. + strength_order = { + None: 0, + VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, + VaSpecStrengthOfEvidenceProvided.MODERATE: 2, + VaSpecStrengthOfEvidenceProvided.STRONG: 3, + VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, + } + + # Collect all calibrations with their evidence strength and criterion + candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] + + for calibration in calibrations: + functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant( + context.variant, calibration + ) + if functional_range is not None: + strength_value = strength_order.get(evidence_strength, 0) + is_benign = criterion.name.startswith("B") if criterion else False + candidates.append((calibration, functional_range, strength_value, is_benign)) + + # If variant is not in any range, return first calibration with None + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Find the maximum strength + max_strength = max(c[2] for c in candidates) + strongest_candidates = [c for c in candidates if c[2] == max_strength] + + # If only one with max strength, return it + if len(strongest_candidates) == 1: + return strongest_candidates[0][0], strongest_candidates[0][1] + + # Tie: check for conflicting evidence (both benign and pathogenic) + has_benign = any(c[3] for c in strongest_candidates) + has_pathogenic = any(not c[3] for c in strongest_candidates) + + # If there's a conflict between benign and pathogenic evidence of equal strength, + # return None for the functional range to indicate uncertain significance + if has_benign and has_pathogenic: + return strongest_candidates[0][0], None + + # Otherwise return the first of the strongest + return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index c7707eb1c..b7904637b 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -6,9 +6,9 @@ from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.variant import Variant logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): def _classification_contains_variant( functional_classification: ScoreCalibrationFunctionalClassification, - mapped_variant: MappedVariant, + variant: Variant, containing_classification_ids: Optional[set[int]], ) -> bool: """Whether this classification's score range contains the variant. @@ -33,11 +33,11 @@ def _classification_contains_variant( """ if containing_classification_ids is not None: return functional_classification.id in containing_classification_ids - return mapped_variant.variant in functional_classification.variants + return variant in functional_classification.variants def functional_classification_of_variant( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: ScoreCalibration, containing_classification_ids: Optional[set[int]] = None, ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: @@ -51,20 +51,20 @@ def functional_classification_of_variant( every range. A caller classifying many variants should resolve membership once from the association table; see ``mavedb.lib.csv.variant``. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify functional impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify functional impact." ) for functional_range in score_calibration.functional_classifications: - if _classification_contains_variant(functional_range, mapped_variant, containing_classification_ids): + if _classification_contains_variant(functional_range, variant, containing_classification_ids): if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -75,7 +75,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: ScoreCalibration, containing_classification_ids: Optional[set[int]] = None, ) -> tuple[ @@ -95,20 +95,20 @@ def pathogenicity_classification_of_variant( Raises ValueError if required calibration, score, or evidence strength is missing. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify clinical impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify clinical impact." ) for pathogenicity_range in score_calibration.functional_classifications: - if _classification_contains_variant(pathogenicity_range, mapped_variant, containing_classification_ids): + if _classification_contains_variant(pathogenicity_range, variant, containing_classification_ids): if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) @@ -147,7 +147,7 @@ def pathogenicity_classification_of_variant( not in VariantPathogenicityEvidenceLine.Criterion._member_names_ ): # pragma: no cover - enforced by model validators in FunctionalClassification view model raise ValueError( - f"Variant {mapped_variant.variant.urn} is contained in a clinical calibration range with an invalid criterion." + f"Variant {variant.urn} is contained in a clinical calibration range with an invalid criterion." " Unable to classify clinical impact." ) diff --git a/src/mavedb/lib/annotation/condition.py b/src/mavedb/lib/annotation/condition.py index cbd5fabe0..b910138f9 100644 --- a/src/mavedb/lib/annotation/condition.py +++ b/src/mavedb/lib/annotation/condition.py @@ -1,21 +1,13 @@ -from ga4gh.core.models import Coding, iriReference as IRI, MappableConcept from ga4gh.va_spec.base.domain_entities import Condition -from mavedb.lib.annotation.constants import GENERIC_DISEASE_MEDGEN_CODE, MEDGEN_SYSTEM +from mavedb.lib.mondo import mondo_term_to_mappable_concept +from mavedb.models.score_calibration import ScoreCalibration -def generic_disease_condition_iri() -> IRI: - return IRI(root=f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}") +def calibration_disease_condition(score_calibration: ScoreCalibration) -> Condition: + """The disease/disorder a calibration applies to, as a VA-Spec ``Condition``. - -def generic_disease_condition() -> Condition: - return Condition( - root=MappableConcept( - conceptType="Disease", - primaryCoding=Coding( - code=GENERIC_DISEASE_MEDGEN_CODE, - system=MEDGEN_SYSTEM, - iris=[generic_disease_condition_iri()], - ), - ) - ) + Every calibration carries a non-null MONDO disease term (often the generic "disease or + disorder" (``MONDO:0000001``)) which is serialized directly to a condition. + """ + return Condition(**mondo_term_to_mappable_concept(score_calibration.disease_term).model_dump(exclude_none=True)) diff --git a/src/mavedb/lib/annotation/conformance.py b/src/mavedb/lib/annotation/conformance.py index 851980cb5..8766a19cb 100644 --- a/src/mavedb/lib/annotation/conformance.py +++ b/src/mavedb/lib/annotation/conformance.py @@ -11,7 +11,7 @@ """ import json -from typing import TypeVar +from typing import Any, TypeVar, Union from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement @@ -21,6 +21,20 @@ ) +# VariantPathogenicityStatement is a Statement, so the union covers every emitted layer. +EmittedAnnotation = Union[ExperimentalVariantFunctionalImpactStudyResult, Statement] + + +def serialize_annotation(annotation: EmittedAnnotation, *, exclude_none: bool = True) -> dict[str, Any]: + """Dump an annotation for emission, nested evidence included in full. + + VA-Spec declares ``EvidenceLine.hasEvidenceItems`` as ``list[InformationEntity]``, and pydantic serializes + a field as its declared type: without ``serialize_as_any``, a nested study result loses ``focus``, + ``functionalImpactScore`` and ``sourceDataSet``. Every emission path goes through here for that reason. + """ + return annotation.model_dump(exclude_none=exclude_none, serialize_as_any=True) + + class AnnotationRoundTripError(Exception): """An emitted annotation did not survive serialization and re-validation.""" @@ -28,8 +42,8 @@ class AnnotationRoundTripError(Exception): def round_trip_annotation(annotation: Annotation) -> Annotation: """Serialize an annotation the way the API emits it, then read it back. - Mirrors the emission path exactly — ``model_dump(exclude_none=True)`` then ``json.dumps(default=str)`` - — because the failures worth catching are the ones those two steps introduce. + Mirrors the emission path exactly — :func:`serialize_annotation` then ``json.dumps(default=str)`` — + because the failures worth catching are the ones those two steps introduce. The assertion is that emission is a fixed point: the re-validated object dumps to the same JSON it was parsed from. Object equality would be the wrong test. VA-Spec declares container fields in terms @@ -44,14 +58,14 @@ def round_trip_annotation(annotation: Annotation) -> Annotation: that serializes differently than what was emitted. """ model = type(annotation) - emitted = json.dumps(annotation.model_dump(exclude_none=True), default=str) + emitted = json.dumps(serialize_annotation(annotation), default=str) try: reparsed = model.model_validate(json.loads(emitted)) except Exception as err: raise AnnotationRoundTripError(f"{model.__name__} did not re-validate after emission: {err}") from err - re_emitted = json.dumps(reparsed.model_dump(exclude_none=True), default=str) + re_emitted = json.dumps(serialize_annotation(reparsed), default=str) if re_emitted != emitted: raise AnnotationRoundTripError(f"{model.__name__} did not survive a round trip unchanged.") diff --git a/src/mavedb/lib/annotation/constants.py b/src/mavedb/lib/annotation/constants.py deleted file mode 100644 index 90b7dfecf..000000000 --- a/src/mavedb/lib/annotation/constants.py +++ /dev/null @@ -1,2 +0,0 @@ -GENERIC_DISEASE_MEDGEN_CODE = "C0012634" -MEDGEN_SYSTEM = "https://www.ncbi.nlm.nih.gov/medgen/" diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py new file mode 100644 index 000000000..eca672cd7 --- /dev/null +++ b/src/mavedb/lib/annotation/context.py @@ -0,0 +1,126 @@ +"""Allele-graph variant context for annotation builders. + +Serves ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate to VA-Spec builders, +and provides a single point of truth for the live (or as-of) mapping record, authoritative allele, +and pre-built VA proposition subject. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional, Sequence + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleCrossReferences, get_allele_cross_references +from mavedb.lib.alleles import LiveRecordLinks, get_live_record_allele_links +from mavedb.lib.cat_vrs import build_categorical_variant, categorical_member_links +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.variant import Variant + + +@dataclass +class VariantAnnotationContext: + """A variant's annotation inputs, sourced from it's allele-graph. + + ``record`` is the live (or as-of) ``MappingRecord``. Its ``mapping_api_version`` / ``mapped_date`` + supply VA provenance, and its ``ValidTime`` is what ``as_of`` and supersession are evaluated against. + ``measured_allele`` is the authoritative allele (the assayed representation) and its ``post_mapped`` VRS is + the concrete study-result focus. ``subject_variant`` is the VA *proposition* subject. When projections + exist, this is a Cat-VRS ``CategoricalVariant`` object. Otherwise, it is the measured ``MolecularVariation``. + """ + + variant: Variant + record: MappingRecord + measured_allele: Allele + subject_variant: MolecularVariation | CategoricalVariant + as_of: Optional[datetime] + + +@dataclass(frozen=True) +class AnnotationContextInputs: + """The database inputs for a chunk of variants' annotation contexts, loaded up front. + + ``record_links`` holds each variant's live (or as-of) record and links; ``cross_references`` holds the + gnomAD and ClinVar identifiers of every allele a proposition subject in the chunk draws on, keyed by + digest. :meth:`context_for` builds one variant's context from these without touching the database, so a + caller can isolate a failure to that variant. + """ + + record_links: dict[int, LiveRecordLinks] + cross_references: dict[str, AlleleCrossReferences] + as_of: Optional[datetime] + + def context_for(self, variant: Variant) -> Optional[VariantAnnotationContext]: + """Build ``variant``'s context, or ``None`` when it is unmapped at ``as_of`` or its authoritative + allele carries no ``post_mapped`` VRS (nothing to annotate).""" + record_links = self.record_links.get(variant.id) + if record_links is None: + return None + + links = record_links.links + measured_allele = next((link.allele for link in links if link.is_authoritative), None) + if measured_allele is None or measured_allele.post_mapped is None: + return None + + # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a + # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* + # (include_convergent=False): the convergent encodings are dropped, because VA-Spec carries no per-member + # provenance to mark them as unmeasured, and StudyResult.focus already pins the concrete measured + # allele. A lone measured allele is served bare. + transit = None + if len(categorical_member_links(links, include_convergent=False)) > 1: + transit = build_categorical_variant( + links, name=variant.urn or "", include_convergent=False, cross_references=self.cross_references + ) + if transit is not None and len(transit.categorical_variant.members or []) > 1: + subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant + else: + subject_variant = vrs_object_from_mapped_variant(measured_allele.post_mapped) + + return VariantAnnotationContext( + variant=variant, + record=record_links.record, + measured_allele=measured_allele, + subject_variant=subject_variant, + as_of=self.as_of, + ) + + +def load_annotation_context_inputs( + db: Session, variants: Sequence[Variant], *, as_of: Optional[datetime] = None +) -> AnnotationContextInputs: + """Load the annotation-context inputs for ``variants`` in at most four queries in total, however many + variants there are (where building each context alone would cost three or four per variant). + + Two for the records and links (:func:`get_live_record_allele_links`), and two for the cross-references + of the alleles in subjects with a projection member, skipped when no subject has one. Whole-set callers pass a chunk of variants + (:data:`~mavedb.lib.alleles.VARIANT_CHUNK_SIZE`). + """ + record_links = get_live_record_allele_links(db, [variant.id for variant in variants], as_of=as_of) + + subject_alleles: list[Allele] = [] + for entry in record_links.values(): + member_links = categorical_member_links(entry.links, include_convergent=False) + if len(member_links) > 1: + subject_alleles.extend(link.allele for link in member_links) + + return AnnotationContextInputs( + record_links=record_links, + cross_references=get_allele_cross_references(db, subject_alleles, as_of=as_of) if subject_alleles else {}, + as_of=as_of, + ) + + +def variant_annotation_context( + db: Session, variant: Variant, *, as_of: Optional[datetime] = None +) -> Optional[VariantAnnotationContext]: + """Assemble the annotation context for one variant from the live (or as-of) mapping substrate. + + The single-variant form of :func:`load_annotation_context_inputs`; see + :meth:`AnnotationContextInputs.context_for` for when it returns ``None``. + """ + return load_annotation_context_inputs(db, [variant], as_of=as_of).context_for(variant) diff --git a/src/mavedb/lib/annotation/contribution.py b/src/mavedb/lib/annotation/contribution.py index 4b0de881f..a63979920 100644 --- a/src/mavedb/lib/annotation/contribution.py +++ b/src/mavedb/lib/annotation/contribution.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime from ga4gh.core.models import Extension from ga4gh.va_spec.base.core import Contribution @@ -9,8 +8,8 @@ mavedb_user_agent, mavedb_vrs_agent, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.types.annotation import ResourceWithCreationModificationDates -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User @@ -21,27 +20,37 @@ def mavedb_api_contribution() -> Contribution: """ Create a [VA Contribution](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/activities/contribution.html#contribution) object for an arbitary contribution from the MaveDB API/software distribution. + + Carries no ``date``. What this contribution asserts is *which software* produced the annotation, and + that is the agent's version — see :func:`mavedb_api_agent`, which stamps ``__version__``. The only + date available at this point is the instant of serialization, which is not provenance: it describes + when a view was rendered, not when anything was contributed. The sibling builders in this module all + carry a real stored date instead. + + Stamping wall-clock time here also made the object unique per call, so identical provenance never + deduplicated and ``va.ndjson`` changed bytes on every run of unchanged data — defeating checksum + manifests and incremental sync for consumers of the public dump. When an artifact needs a generation + time it belongs on that artifact's metadata, once, not on every nested object inside it. """ return Contribution( name="MaveDB API", description="Contribution from the MaveDB API", contributor=mavedb_api_agent(), - date=datetime.today(), activityType="software application programming interface", ) -def mavedb_vrs_contribution(mapped_variant: MappedVariant) -> Contribution: +def mavedb_vrs_contribution(context: VariantAnnotationContext) -> Contribution: """ Create a [VA Contribution](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/activities/contribution.html#contribution) - object from the provided mapped variant. + object from the variant's live mapping record (the VRS mapper's provenance). """ return Contribution( name="MaveDB VRS Mapper", description="Contribution from the MaveDB VRS mapping software", - # Guaranteed to be a str via DB constraints. - contributor=mavedb_vrs_agent(mapped_variant.mapping_api_version), # type: ignore - date=mapped_variant.mapped_date, # type: ignore + # Both guaranteed non-null via DB constraints on MappingRecord. + contributor=mavedb_vrs_agent(context.record.mapping_api_version), # type: ignore + date=context.record.mapped_date, # type: ignore activityType="human genome sequence mapping process", ) diff --git a/src/mavedb/lib/annotation/document.py b/src/mavedb/lib/annotation/document.py index 039142030..d9e58cfc2 100644 --- a/src/mavedb/lib/annotation/document.py +++ b/src/mavedb/lib/annotation/document.py @@ -7,8 +7,8 @@ from ga4gh.va_spec.base.core import Document from mavedb.constants import MAVEDB_FRONTEND_URL +from mavedb.models.allele import Allele from mavedb.models.experiment import Experiment -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -128,31 +128,34 @@ def score_calibration_as_document(score_calibration: ScoreCalibration) -> Docume ) -def mapped_variant_as_iri(mapped_variant: MappedVariant) -> Optional[IRI]: +def measured_allele_as_iri(allele: Allele) -> Optional[IRI]: """ - Create an IRI as described in for the provided MaveDB mapped variant. Within - the context of VA-Spec, these can be used interchangeably with an equivalent document object for brevity. + Create an IRI as described in for the + measured (authoritative) allele, keyed by its ClinGen allele id. Within the context of VA-Spec, these + can be used interchangeably with an equivalent document object for brevity. ``None`` when the allele + has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None - return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(mapped_variant.clingen_allele_id)}") + return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}") -def mapped_variant_to_document(mapped_variant: MappedVariant) -> Optional[Document]: +def measured_allele_to_document(allele: Allele) -> Optional[Document]: """ Create a [VA Document](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/information-entities/document.html#document) - object from the provided MaveDB mapped variant. + object from the provided MaveDB measured (authoritative) allele, keyed by its ClinGen allele id. ``None`` + when the allele has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None return Document( - id=mapped_variant.variant.urn, - name="MaveDB Mapped Variant", - documentType="mapped genomic variant description", - # We only reach this point if a IRI is guaranteed to exist - urls=[mapped_variant_as_iri(mapped_variant).root], # type: ignore + id=allele.clingen_allele_id, + name="MaveDB Measured Allele", + documentType="measured allele", + # The measured allele document is keyed by its ClinGen allele id, so the IRI is guaranteed to exist. + urls=[measured_allele_as_iri(allele).root], # type: ignore ) diff --git a/src/mavedb/lib/annotation/eligibility.py b/src/mavedb/lib/annotation/eligibility.py new file mode 100644 index 000000000..e4c029694 --- /dev/null +++ b/src/mavedb/lib/annotation/eligibility.py @@ -0,0 +1,108 @@ +"""Whether a variant is eligible for VA-Spec annotation. + +Gate predicates the annotation builders consult before constructing a statement: a variant needs a real +score (the base assumption) and its score set needs at least one calibration usable for the target +annotation type (:func:`calibration.calibrations_available_for_annotation`). This calibration must be readable +by the requesting principal. Split by annotation type into ``can_annotate_variant_for_pathogenicity_evidence`` / +``can_annotate_variant_for_functional_statement``. +""" + +from typing import Optional + +from mavedb.lib.annotation.calibration import calibrations_available_for_annotation +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.variants import variant_score + + +def _can_annotate_variant_base_assumptions(context: VariantAnnotationContext) -> bool: + """ + Check if a variant meets the basic requirements for annotation. + + This function validates that a variant has the necessary data to proceed with + annotation by checking for a valid score value. + + Args: + context (VariantAnnotationContext): The context containing the variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated (has a non-None numeric score), False otherwise. + """ + # A variant is annotatable only if it carries a real (non-null, numeric) score. + if variant_score(context.variant) is None: + return False + + return True + + +def can_annotate_variant_for_pathogenicity_evidence( + context: VariantAnnotationContext, + allow_research_use_only_calibrations=False, + principal: Optional[Principal] = None, +) -> bool: + """ + Determine if a variant can be annotated for pathogenicity evidence. + + This function checks if a variant meets all the necessary conditions to receive + pathogenicity evidence annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for pathogenicity evidence annotation. + + Args: + context (VariantAnnotationContext): The context containing the variant to evaluate for pathogenicity evidence annotation eligibility. + + Returns: + bool: True if the variant can be annotated for pathogenicity evidence, False otherwise. + + Notes: + The function performs two main validation checks: + 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. + + Both checks must pass for the variant to be considered eligible for + pathogenicity evidence annotation. + """ + if not _can_annotate_variant_base_assumptions(context): + return False + + return bool( + calibrations_available_for_annotation( + context, + "pathogenicity", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) + + +def can_annotate_variant_for_functional_statement( + context: VariantAnnotationContext, allow_research_use_only_calibrations=False, principal: Optional[Principal] = None +) -> bool: + """ + Determine if a variant can be annotated for functional statements. + + This function checks if a variant meets all the necessary conditions to receive + functional annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for functional annotation. + + Args: + context (VariantAnnotationContext): The context containing the variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated for functional statements, False otherwise. + + Notes: + The function performs two main checks: + 1. Validates base assumptions using _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for functional annotation. + """ + if not _can_annotate_variant_base_assumptions(context): + return False + + return bool( + calibrations_available_for_annotation( + context, + "functional", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index d30e90542..00e5823d4 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -5,8 +5,7 @@ from ga4gh.va_spec.base.core import ( Direction, EvidenceLine, - EvidenceLineType, - StatementType, + Statement, StudyResult, VariantPathogenicityProposition, ) @@ -16,6 +15,7 @@ functional_classification_of_variant, pathogenicity_classification_of_variant, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -30,18 +30,17 @@ functional_score_calibration_as_method, pathogenicity_score_calibration_as_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration def acmg_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, proposition: VariantPathogenicityProposition, - evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], + evidence: list[Union[StudyResult, EvidenceLine, Statement, iriReference]], ) -> VariantPathogenicityEvidenceLine: containing_evidence_range, evidence_outcome, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, score_calibration + context.variant, score_calibration ) evidence_outcome_code = acmg_evidence_outcome_code( @@ -61,9 +60,9 @@ def acmg_evidence_line( direction_of_evidence = direction_of_support_for_pathogenicity_classification(evidence_outcome) return VariantPathogenicityEvidenceLine( - description=f"Pathogenicity evidence line for {mapped_variant.variant.urn}.", + description=f"Pathogenicity evidence line for {context.variant.urn}.", hasEvidenceItems=list(evidence), - specifiedBy=pathogenicity_score_calibration_as_method(score_calibration, evidence_outcome), + specifiedBy=pathogenicity_score_calibration_as_method(score_calibration), evidenceOutcome={ "primaryCoding": Coding( code=evidence_outcome_code, @@ -75,7 +74,7 @@ def acmg_evidence_line( directionOfEvidenceProvided=direction_of_evidence, contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], targetProposition=proposition, @@ -91,26 +90,26 @@ def acmg_evidence_line( def functional_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, - evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], + evidence: list[Union[StudyResult, EvidenceLine, Statement, iriReference]], ) -> EvidenceLine: - containing_evidence_range, classification = functional_classification_of_variant(mapped_variant, score_calibration) + containing_evidence_range, classification = functional_classification_of_variant(context.variant, score_calibration) return EvidenceLine( - description=f"Functional evidence line for {mapped_variant.variant.urn}", - hasEvidenceItems=[StudyResult(root=item) for item in evidence], + description=f"Functional evidence line for {context.variant.urn}", + hasEvidenceItems=list(evidence), directionOfEvidenceProvided=direction_of_support_for_functional_classification(classification), evidenceOutcome=MappableConcept( primaryCoding=Coding( code=classification.value, - system="ga4gh-gks-term:experimental-var-func-impact-classification", + system="ga4gh-gkm-term:experimental-var-func-impact-classification", ), ), specifiedBy=functional_score_calibration_as_method(score_calibration), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], reportedIn=[score_calibration_as_document(score_calibration)], diff --git a/src/mavedb/lib/annotation/flatten.py b/src/mavedb/lib/annotation/flatten.py index 04d2506f7..5ffe06377 100644 --- a/src/mavedb/lib/annotation/flatten.py +++ b/src/mavedb/lib/annotation/flatten.py @@ -14,8 +14,8 @@ functional_classification_of_variant, pathogenicity_classification_of_variant, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.variant import Variant @dataclass(frozen=True) @@ -56,7 +56,7 @@ class FlatAnnotation: def flatten_annotation( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: Optional[ScoreCalibration], containing_classification_ids: Optional[set[int]] = None, ) -> FlatAnnotation: @@ -86,7 +86,7 @@ def flatten_annotation( ) _, functional_classification = functional_classification_of_variant( - mapped_variant, score_calibration, containing_classification_ids + variant, score_calibration, containing_classification_ids ) functional_only = FlatAnnotation( @@ -103,7 +103,7 @@ def flatten_annotation( # VA-Spec strength deliberately unused: it has already collapsed MODERATE_PLUS to MODERATE. containing_range, criterion, _va_spec_evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, score_calibration, containing_classification_ids + variant, score_calibration, containing_classification_ids ) # Read the strength off the containing range, which keeps MODERATE_PLUS. Reachable in practice: diff --git a/src/mavedb/lib/annotation/method.py b/src/mavedb/lib/annotation/method.py index 1d7f0941a..f8d3801b1 100644 --- a/src/mavedb/lib/annotation/method.py +++ b/src/mavedb/lib/annotation/method.py @@ -143,12 +143,13 @@ def pathogenicity_score_calibration_as_iri(score_calibration: ScoreCalibration) return publication_as_iri(publication) if publication else None -def pathogenicity_score_calibration_as_method( - score_calibration: ScoreCalibration, method: Optional[VariantPathogenicityEvidenceLine.Criterion] -) -> Method: +def pathogenicity_score_calibration_as_method(score_calibration: ScoreCalibration) -> Method: """ Generate a [VA Method](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/information-entities/method.html#method) object for a generic calibration. + + A calibration assesses functional data, so its method type is always ``functional_data_assessment``, + the ACMG 2015 type that permits the PS3 and BS3 criteria its evidence lines carry. """ # TODO#XXX - in a future software version, it will be required that the method is populated with an IRI or equivalent. # Currently, we populate the method with an IRI if a publication with the appropriate relationship is found, @@ -157,7 +158,7 @@ def pathogenicity_score_calibration_as_method( return Method( name="Calibration method", reportedIn=pathogenicity_score_calibration_as_iri(score_calibration) or "Not Provided", - methodType=method.value if method else None, + methodType=VariantPathogenicityEvidenceLine.MethodType.FUNCTIONAL_DATA_ASSESSMENT.value, ) diff --git a/src/mavedb/lib/annotation/proposition.py b/src/mavedb/lib/annotation/proposition.py index 963327c73..440a6f331 100644 --- a/src/mavedb/lib/annotation/proposition.py +++ b/src/mavedb/lib/annotation/proposition.py @@ -1,43 +1,100 @@ from ga4gh.core.models import Coding, MappableConcept from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition -from mavedb.lib.annotation.condition import generic_disease_condition +from mavedb.lib.annotation.condition import calibration_disease_condition +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.document import experiment_to_document -from mavedb.lib.annotation.util import sequence_feature_for_mapped_variant, variation_from_mapped_variant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata +from mavedb.lib.types.annotation import SequenceFeature +from mavedb.lib.variants import target_for_variant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.variant import Variant -def mapped_variant_to_experimental_variant_clinical_impact_proposition( - mapped_variant: MappedVariant, +def sequence_feature_for_variant(variant: Variant) -> SequenceFeature: + """ + Extract the sequence feature (e.g., gene or transcript) associated with a variant. + + This function retrieves the sequence feature from the variant's target data, which is + necessary for generating annotations that reference specific genomic features. Co-located with the + propositions that consume it — its only caller. + + Args: + variant (Variant): The variant whose target gene supplies the sequence feature. + + Returns: + SequenceFeature: Named tuple with: + - `identifier`: sequence feature identifier (gene/transcript ID or name) + - `system`: source/system URL for the identifier + + """ + target = target_for_variant(variant) + if target is None: + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable target gene." + " Unable to extract sequence feature for annotation." + ) + + # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. + # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or + # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. + if target.mapped_hgnc_name: + return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") + + post_mapped_ids = extract_ids_from_post_mapped_metadata( + target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore + ) + if post_mapped_ids: + post_mapped_id = post_mapped_ids[0] + if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): + return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") + elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): + return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") + + return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") + + if target.name: + return SequenceFeature(target.name, "https://www.mavedb.org/") + + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable sequence feature in its target gene data." + " Unable to extract sequence feature for annotation." + ) + + +def variant_pathogenicity_proposition( + context: VariantAnnotationContext, + score_calibration: ScoreCalibration, ) -> VariantPathogenicityProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return VariantPathogenicityProposition( - description=f"Variant pathogenicity proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant pathogenicity proposition for {context.variant.urn}.", + subject=context.subject_variant, predicate="isCausalFor", - objectCondition=generic_disease_condition(), + object=calibration_disease_condition(score_calibration), geneContextQualifier=sequence_feature if system == "https://www.genenames.org/" else None, # only include gene context if we have a gene identifier ) -def mapped_variant_to_experimental_variant_functional_impact_proposition( - mapped_variant: MappedVariant, +def variant_functional_impact_proposition( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return ExperimentalVariantFunctionalImpactProposition( - description=f"Variant functional impact proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant functional impact proposition for {context.variant.urn}.", + subject=context.subject_variant, predicate="impactsFunctionOf", - objectSequenceFeature=sequence_feature, - experimentalContextQualifier=experiment_to_document(mapped_variant.variant.score_set.experiment), + object=sequence_feature, + experimentalContextQualifier=experiment_to_document(context.variant.score_set.experiment), ) diff --git a/src/mavedb/lib/annotation/statement.py b/src/mavedb/lib/annotation/statement.py index c77d30f1a..d897ef20a 100644 --- a/src/mavedb/lib/annotation/statement.py +++ b/src/mavedb/lib/annotation/statement.py @@ -10,6 +10,7 @@ ) from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -20,23 +21,22 @@ variant_interpretation_functional_guideline_method, variant_interpretation_pathogenicity_guideline_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification -def mapped_variant_to_functional_statement( - mapped_variant: MappedVariant, +def functional_statement( + context: VariantAnnotationContext, proposition: ExperimentalVariantFunctionalImpactProposition, evidence: list[EvidenceLine], score_calibration: ScoreCalibration, functional_classification: ExperimentalVariantFunctionalImpactClassification, ) -> Statement: """ - Create a functional impact statement for a mapped variant. + Create a functional impact statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The functional impact proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -48,11 +48,11 @@ def mapped_variant_to_functional_statement( direction = aggregate_direction_of_evidence(evidence) return Statement( - description=f"Variant functional impact statement for {mapped_variant.variant.urn}.", + description=f"Variant functional impact statement for {context.variant.urn}.", specifiedBy=variant_interpretation_functional_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, @@ -60,25 +60,25 @@ def mapped_variant_to_functional_statement( classification=MappableConcept( primaryCoding=Coding( code=functional_classification, - system="ga4gh-gks-term:experimental-var-func-impact-classification", + system="ga4gh-gkm-term:experimental-var-func-impact-classification", ), ), hasEvidenceLines=[evidence_item for evidence_item in evidence], ) -def mapped_variant_to_pathogenicity_statement( - mapped_variant: MappedVariant, +def pathogenicity_statement( + context: VariantAnnotationContext, proposition: VariantPathogenicityProposition, evidence: list[VariantPathogenicityEvidenceLine], score_calibration: ScoreCalibration, functional_range: Optional[ScoreCalibrationFunctionalClassification], ) -> VariantPathogenicityStatement: """ - Create a pathogenicity statement for a mapped variant. + Create a pathogenicity statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The pathogenicity proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -105,11 +105,11 @@ def mapped_variant_to_pathogenicity_statement( acmg_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE return VariantPathogenicityStatement( - description=f"Variant pathogenicity statement for {mapped_variant.variant.urn}.", + description=f"Variant pathogenicity statement for {context.variant.urn}.", specifiedBy=variant_interpretation_pathogenicity_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, diff --git a/src/mavedb/lib/annotation/study_result.py b/src/mavedb/lib/annotation/study_result.py index 85dbcbdde..cdf4d160f 100644 --- a/src/mavedb/lib/annotation/study_result.py +++ b/src/mavedb/lib/annotation/study_result.py @@ -1,5 +1,6 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_creator_contribution, @@ -7,30 +8,31 @@ mavedb_vrs_contribution, ) from mavedb.lib.annotation.dataset import score_set_to_data_set -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri from mavedb.lib.annotation.method import ( publication_identifiers_to_method, ) -from mavedb.lib.annotation.util import variation_from_mapped_variant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.lib.variants import variant_score -def mapped_variant_to_experimental_variant_impact_study_result( - mapped_variant: MappedVariant, +def variant_impact_study_result( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactStudyResult: + # The study result's focus is the concrete measured allele — never a CategoricalVariant (VA-Spec + # narrows ``focus`` to MolecularVariation). The context guarantees a hydratable post_mapped. return ExperimentalVariantFunctionalImpactStudyResult( - description=f"Variant effect study result for {mapped_variant.variant.urn}.", - focusVariant=variation_from_mapped_variant(mapped_variant), - functionalImpactScore=mapped_variant.variant.data["score_data"]["score"], # type: ignore - specifiedBy=publication_identifiers_to_method( - mapped_variant.variant.score_set.publication_identifier_associations - ), - sourceDataSet=score_set_to_data_set(mapped_variant.variant.score_set), + description=f"Variant effect study result for {context.variant.urn}.", + # post_mapped is guaranteed non-null by variant_annotation_context (it returns None otherwise). + focus=vrs_object_from_mapped_variant(context.measured_allele.post_mapped), # type: ignore[arg-type] + functionalImpactScore=variant_score(context.variant), + specifiedBy=publication_identifiers_to_method(context.variant.score_set.publication_identifier_associations), + sourceDataSet=score_set_to_data_set(context.variant.score_set), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), - mavedb_creator_contribution(mapped_variant.variant, mapped_variant.variant.score_set.created_by), - mavedb_modifier_contribution(mapped_variant.variant, mapped_variant.variant.score_set.modified_by), + mavedb_vrs_contribution(context), + mavedb_creator_contribution(context.variant, context.variant.score_set.created_by), + mavedb_modifier_contribution(context.variant, context.variant.score_set.modified_by), ], - reportedIn=filter(None, [variant_as_iri(mapped_variant.variant), mapped_variant_as_iri(mapped_variant)]), + reportedIn=filter(None, [variant_as_iri(context.variant), measured_allele_as_iri(context.measured_allele)]), ) diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py deleted file mode 100644 index 23ae95b66..000000000 --- a/src/mavedb/lib/annotation/util.py +++ /dev/null @@ -1,571 +0,0 @@ -from typing import Iterable, Literal, Optional - -from ga4gh.core.models import Extension -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided -from ga4gh.vrs.models import ( - Allele, - CisPhasedBlock, - Expression, - LengthExpression, - LiteralSequenceExpression, - MolecularVariation, - ReferenceLengthExpression, - SequenceLocation, - SequenceReference, -) - -from mavedb.lib.annotation.classification import ( - ExperimentalVariantFunctionalImpactClassification, - functional_classification_of_variant, - pathogenicity_classification_of_variant, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata -from mavedb.lib.permissions.principal import Principal -from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.types.annotation import SequenceFeature -from mavedb.lib.variants import target_for_variant -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_calibration import ScoreCalibration -from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification - -CALIBRATION_SCOPE_EXTENSION_NAME = "mavedb_calibration_scope" -"""Extension naming the principal an annotation was built for. See ``calibration_scope_extension``.""" - - -def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: - """ - Converts a dictionary containing allelic mapping results into an Allele object. - - This function handles the possibility of an extra nesting level in early VRS 1.3 objects, - where Allele objects are contained within a `variation` property. If the `variation` key - is not present, the function assumes the dictionary itself represents the variation. - - Args: - allelic_mapping_results (dict): A dictionary containing allelic mapping results. - It may include a `variation` key or directly represent the variation. - - Returns: - Allele: An Allele object constructed from the provided mapping results. - - Raises: - KeyError: If required keys are missing from the input dictionary. - """ - - # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects - # are contained in a `variation` property. Although it's unlikely variants of this form - # will ever be exported in this format, we handle the possibility. - try: - variation = allelic_mapping_results["variation"] - except KeyError: - variation = allelic_mapping_results - - state_dict = variation["state"] - state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression - if state_dict.get("type") == "ReferenceLengthExpression": - state = ReferenceLengthExpression(**state_dict) - elif state_dict.get("type") == "LengthExpression": - state = LengthExpression(**state_dict) - elif state_dict.get("type") == "LiteralSequenceExpression": - state = LiteralSequenceExpression(**state_dict) - else: - raise ValueError( - f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " - "Update allele_from_mapped_variant_dictionary_result to handle this type." - ) - - # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on - # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid, so using model_validate() - # directly is not possible against stored mapping results. - return Allele( - id=variation.get("id"), - state=state, - digest=variation.get("digest"), - location=SequenceLocation( - start=variation.get("location", {}).get("start"), - end=variation.get("location", {}).get("end"), - digest=variation.get("location", {}).get("digest"), - id=variation.get("location", {}).get("id"), - sequenceReference=SequenceReference( - name=variation.get("location", {}).get("sequenceReference", {}).get("name"), - refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), - ), - ), - extensions=[ - Extension( - id=extension.get("id"), - name=extension["name"], - description=extension.get("description"), - value=extension.get("value"), - ) - for extension in variation.get("extensions", []) - ], - expressions=[ - Expression( - id=expression.get("id"), - syntax=expression["syntax"], - syntax_version=expression.get("syntax_version"), - value=expression["value"], - ) - for expression in variation.get("expressions", []) - ], - ) - - -def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: - """ - Extracts a VRS (Variation Representation Specification) object from a mapped variant. - - This function processes a dictionary of mapping results and returns a VRS object, - which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object - returned depends on the "type" field in the input dictionary. - - Args: - mapping_results (dict): A dictionary containing the mapping results of a variant. - It must include a "type" key indicating the type of VRS object - ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). - If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also - include a "members" key containing a list of member alleles. - - Returns: - MolecularVariation: A VRS object representing the mapped variant. This will be - either a `CisPhasedBlock` containing its member variants or an `Allele` - derived from the mapping results. - - Raises: - KeyError: If required keys are missing from the `mapping_results` dictionary. - """ - if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": - return MolecularVariation( - # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) - CisPhasedBlock( - id=None, - members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], - ) - ) - - return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) - - -def variation_from_mapped_variant(mapped_variant: MappedVariant) -> MolecularVariation: - """ - Converts mapping results from a mapped variant into a MolecularVariation object. - - This function takes a `MappedVariant` object and extracts its post-mapped - variant to generate a corresponding `MolecularVariation` object. If the - `post_mapped` attribute of the `MappedVariant` is `None`, an exception is raised. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing - the variant data. - - Returns: - MolecularVariation: The molecular variation derived from the post-mapped - variant. - - Raises: - MappingDataDoesntExistException: If the `post_mapped` attribute of the - `mapped_variant` is `None`, indicating that the post-mapped variant - data is unavailable. - """ - if mapped_variant.post_mapped is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have a post mapped variant." - " Unable to extract variation data." - ) - - return vrs_object_from_mapped_variant(mapped_variant.post_mapped) - - -def _can_annotate_variant_base_assumptions(mapped_variant: MappedVariant) -> bool: - """ - Check if a mapped variant meets the basic requirements for annotation. - - This function validates that a mapped variant has the necessary data - to proceed with annotation by checking for a valid score value. - - Args: - mapped_variant (MappedVariant): The mapped variant to check for - annotation eligibility. - - Returns: - bool: True if the variant can be annotated (has score ranges and - a non-None score), False otherwise. - """ - # This property is guaranteed to exist for all variants. - if mapped_variant.variant.data["score_data"]["score"] is None: # type: ignore - return False - - return True - - -def score_calibration_may_be_used_for_annotation( - score_calibration: ScoreCalibration, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, -) -> bool: - """ - Check if a score calibration may be used for annotation based on its properties. - - This function evaluates whether a given score calibration is suitable for use in - annotation by checking its research use only status and the presence of required - classifications based on the annotation type. - - Args: - score_calibration (ScoreCalibration): The score calibration to evaluate. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation - being considered, which determines the required classifications for validity. - allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations - marked as research use only for annotation. Defaults to False. - - Returns: - bool: True if the score calibration can be used for annotation, False otherwise. - """ - if score_calibration.research_use_only and not allow_research_use_only_calibrations: - return False - - if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: - return False - - if annotation_type == "pathogenicity" and all( - fr.acmg_classification is None for fr in score_calibration.functional_classifications - ): - return False - - return True - - -def calibrations_available_for_annotation( - mapped_variant: MappedVariant, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, - principal: Optional[Principal] = None, -) -> list[ScoreCalibration]: - """ - Select the calibrations on a mapped variant's score set that may build the requested annotation. - - Two independent questions decide this, and they are asked by different collaborators. - - Eligibility: Does this calibration carry the classifications the annotation type needs, and is - its research-use-only standing permitted here. This is ``score_calibration_may_be_used_for_annotation``. - - Visibility: May this principal read it at all. This is the viewer's, because a calibration's READ rule - is stricter than its score set's. - - Args: - mapped_variant (MappedVariant): The mapped variant whose score set's calibrations are considered. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to be built. - allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use - only are eligible. Defaults to False. - principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous - caller, so a function that omits it gets public calibrations only. - - Returns: - list[ScoreCalibration]: The eligible, visible calibrations, in score set order. - """ - viewer = (principal if principal is not None else Principal()).viewer_for(ScoreCalibrationViewer) - - return [ - score_calibration - for score_calibration in viewer.visible(mapped_variant.variant.score_set.score_calibrations) - if score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type, - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ) - ] - - -def calibration_scope_extension(calibrations: Iterable[ScoreCalibration]) -> Extension: - """ - Describe the principal an annotation was built for, given the calibrations behind it. - - VA-Spec statements carry no stable identifier, so two callers can receive materially different - statements from the same URL. Naming the scope on the object itself is what keeps that honest: a - consumer holding a record can tell whether it is the one anyone would get, or one widened by the - requester's own access. - - Args: - calibrations (Iterable[ScoreCalibration]): The calibrations contributing evidence to the annotation. - - Returns: - Extension: A ``mavedb_calibration_scope`` extension, ``restricted`` when any contributing - calibration is private and ``public`` otherwise. - """ - if any(calibration.private for calibration in calibrations): - return Extension( - name=CALIBRATION_SCOPE_EXTENSION_NAME, - value="restricted", - description=( - "Built from at least one private score calibration, visible to the requesting viewer. " - "Another viewer requesting this variant may receive fewer evidence lines, or none." - ), - ) - - return Extension( - name=CALIBRATION_SCOPE_EXTENSION_NAME, - value="public", - description=( - "Built only from public score calibrations. Any viewer requesting this variant receives the same evidence." - ), - ) - - -def can_annotate_variant_for_pathogenicity_evidence( - mapped_variant: MappedVariant, - allow_research_use_only_calibrations=False, - principal: Optional[Principal] = None, -) -> bool: - """ - Determine if a mapped variant can be annotated for pathogenicity evidence. - - This function checks if a variant meets all the necessary conditions to receive - pathogenicity evidence annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for pathogenicity evidence annotation. - - Args: - mapped_variant (MappedVariant): The mapped variant object to evaluate - for pathogenicity evidence annotation eligibility. - allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use - only are eligible. Defaults to False. - principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous - caller. Must match the principal the annotation itself will be built for, or this answers a - different question than the one the caller is about to act on. - - Returns: - bool: True if the variant can be annotated for pathogenicity evidence, - False otherwise. - - Notes: - The function performs two main validation checks: - 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. - - Both checks must pass for the variant to be considered eligible for - pathogenicity evidence annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - - return bool( - calibrations_available_for_annotation( - mapped_variant, - "pathogenicity", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - principal=principal, - ) - ) - - -def can_annotate_variant_for_functional_statement( - mapped_variant: MappedVariant, - allow_research_use_only_calibrations=False, - principal: Optional[Principal] = None, -) -> bool: - """ - Determine if a mapped variant can be annotated for functional statements. - - This function checks if a variant meets all the necessary conditions to receive - functional annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for functional annotation. - - Args: - mapped_variant (MappedVariant): The variant object to check for annotation - eligibility, containing mapping information and score data. - allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use - only are eligible. Defaults to False. - principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous - caller. Must match the principal the annotation itself will be built for, or this answers a - different question than the one the caller is about to act on. - - Returns: - bool: True if the variant can be annotated for functional statements, - False otherwise. - - Notes: - The function performs two main checks: - 1. Validates base assumptions using _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for functional annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - - return bool( - calibrations_available_for_annotation( - mapped_variant, - "functional", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - principal=principal, - ) - ) - - -def sequence_feature_for_mapped_variant(mapped_variant: MappedVariant) -> SequenceFeature: - """ - Extract the sequence feature (e.g., gene or transcript) associated with a mapped variant. - - This function retrieves the sequence feature from the mapped variant's data, which is - necessary for generating annotations that reference specific genomic features. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant data. - - Returns: - SequenceFeature: Named tuple with: - - `identifier`: sequence feature identifier (gene/transcript ID or name) - - `system`: source/system URL for the identifier - - """ - target = target_for_variant(mapped_variant.variant) - if target is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable target gene." - " Unable to extract sequence feature for annotation." - ) - - # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. - # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or - # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. - if target.mapped_hgnc_name: - return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") - - post_mapped_ids = extract_ids_from_post_mapped_metadata( - target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore - ) - if post_mapped_ids: - post_mapped_id = post_mapped_ids[0] - if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): - return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") - elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): - return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") - - return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") - - if target.name: - return SequenceFeature(target.name, "https://www.mavedb.org/") - - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable sequence feature in its target gene data." - " Unable to extract sequence feature for annotation." - ) - - -def select_strongest_functional_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for functional classification. - - In case of ties or conflicting classifications, defaults to normal classification. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should be classified as INDETERMINATE but still receive annotations. - """ - if not calibrations: - return None, None - - # Collect all calibrations and their classifications - candidates: list[ - tuple[ - ScoreCalibration, - ScoreCalibrationFunctionalClassification, - ExperimentalVariantFunctionalImpactClassification, - ] - ] = [] - - for calibration in calibrations: - functional_range, classification = functional_classification_of_variant(mapped_variant, calibration) - if functional_range is not None: - candidates.append((calibration, functional_range, classification)) - - # If variant is not in any range, return first calibration with None to indicate INDETERMINATE - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Check if all classifications agree - classifications = [c[2] for c in candidates] - if all(cls == classifications[0] for cls in classifications): - # All agree, return the first one - return candidates[0][0], candidates[0][1] - - # Conflict exists: default to normal classification - normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] - if normal_candidates: - return normal_candidates[0][0], normal_candidates[0][1] - - # If no normal classification, return the first candidate - return candidates[0][0], candidates[0][1] - - -def select_strongest_pathogenicity_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for pathogenicity classification. - - Uses ACMG evidence strength to determine the strongest evidence. - In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain - significance by returning None for the functional range. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should receive annotations even though it's not classified in any range. - """ - if not calibrations: - return None, None - - # Define evidence strength ordering (higher index = stronger evidence) - # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. - # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. - strength_order = { - None: 0, - VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, - VaSpecStrengthOfEvidenceProvided.MODERATE: 2, - VaSpecStrengthOfEvidenceProvided.STRONG: 3, - VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, - } - - # Collect all calibrations with their evidence strength and criterion - candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] - - for calibration in calibrations: - functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, calibration - ) - if functional_range is not None: - strength_value = strength_order.get(evidence_strength, 0) - is_benign = criterion.name.startswith("B") if criterion else False - candidates.append((calibration, functional_range, strength_value, is_benign)) - - # If variant is not in any range, return first calibration with None - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Find the maximum strength - max_strength = max(c[2] for c in candidates) - strongest_candidates = [c for c in candidates if c[2] == max_strength] - - # If only one with max strength, return it - if len(strongest_candidates) == 1: - return strongest_candidates[0][0], strongest_candidates[0][1] - - # Tie: check for conflicting evidence (both benign and pathogenic) - has_benign = any(c[3] for c in strongest_candidates) - has_pathogenic = any(not c[3] for c in strongest_candidates) - - # If there's a conflict between benign and pathogenic evidence of equal strength, - # return None for the functional range to indicate uncertain significance - if has_benign and has_pathogenic: - return strongest_candidates[0][0], None - - # Otherwise return the first of the strongest - return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/annotation_status_manager.py b/src/mavedb/lib/annotation_status_manager.py index 6598bfabd..4022cb4d2 100644 --- a/src/mavedb/lib/annotation_status_manager.py +++ b/src/mavedb/lib/annotation_status_manager.py @@ -1,207 +1,170 @@ -"""Manage annotation statuses for variants. - -This module provides functionality to insert and retrieve annotation statuses -for genetic variants, ensuring that only one current status exists per -(variant, annotation type, version) combination. +"""Append-only writer for the annotation event log. + +Buffers :class:`AnnotationEvent` rows and flushes them in batches. This is an +**append-only** log, not a state table: there is no ``current`` flag to +maintain and no retire-on-write step. "Current" is derived at read time +(``DISTINCT ON (subject, annotation_type) … ORDER BY id DESC``) — exposed as +the ``v_current_annotation_events`` view (``mavedb.models.annotation_event_view``). + +Each event's *subject* is either a variant or an allele, chosen by +``annotation_type``. The writer validates the subject/type pairing up front so +a mis-subjected event fails with a clear ``ValueError`` rather than a deferred +DB ``CHECK`` violation at flush. """ import logging from typing import Optional -from sqlalchemy import select, update +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.sql import desc from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.annotation_event import ALLELE_SUBJECT_TYPES, VARIANT_SUBJECT_TYPES, AnnotationEvent logger = logging.getLogger(__name__) -# Default number of pending annotations to accumulate before auto-flushing. +# Default number of pending events to accumulate before auto-flushing. DEFAULT_BATCH_SIZE = 500 class AnnotationStatusManager: - """ - Manager for handling variant annotation statuses with batched writes. + """Buffered, append-only writer for :class:`AnnotationEvent` rows. - Annotations are accumulated in memory and flushed to the database in - batches (default 500) to reduce round-trips. Callers **must** call - :meth:`flush` after the last ``add_annotation`` to persist any remainder. + Events are accumulated in memory and flushed in batches (default 500) to + reduce round-trips. Callers **must** call :meth:`flush` after the last + :meth:`record_event` to persist any remainder. """ - def __init__(self, session: Session, job_run_id: Optional[int] = None, *, batch_size: int = DEFAULT_BATCH_SIZE): + def __init__( + self, + session: Session, + job_run_id: Optional[int] = None, + *, + score_set_id: Optional[int] = None, + batch_size: int = DEFAULT_BATCH_SIZE, + ): self.session = session self.job_run_id = job_run_id + self.score_set_id = score_set_id self.batch_size = batch_size - self._pending: list[VariantAnnotationStatus] = [] - self._retirement_filters: list[dict] = [] + self._pending: list[AnnotationEvent] = [] - def add_annotation( + def record_event( self, - variant_id: int, annotation_type: AnnotationType, - status: AnnotationStatus, - version: Optional[str] = None, - failure_category: Optional[AnnotationFailureCategory] = None, - annotation_data: dict = {}, - current: bool = True, - replace_all_versions: bool = True, + *, + disposition: Disposition, + reason: str, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + metadata: Optional[dict] = None, + score_set_id: Optional[int] = None, ) -> None: - """ - Stage a new annotation and schedule retirement of previous current rows. - - By default (``replace_all_versions=True``), all existing current annotations for - (variant, type) are retired regardless of version. + """Buffer one terminal observation about a variant or an allele. - When ``replace_all_versions=False``, only existing current annotations matching - (variant, type, version) are retired. + Exactly one of ``variant_id`` / ``allele_id`` must be set, and it must + match the subject the ``annotation_type`` keys on. ``reason`` is the + single "what happened" axis across all dispositions (see EventReason). - Writes are accumulated in memory and flushed to the database when - ``batch_size`` is reached. Call :meth:`flush` after the last add to - persist any remaining annotations. - - NOTE: - This method does not commit the session. The caller is responsible - for persisting changes (e.g., via ``session.commit()``). + Writes are accumulated in memory and flushed when ``batch_size`` is + reached. Call :meth:`flush` after the last call to persist the + remainder. Does not commit — the caller owns the transaction. """ - self._retirement_filters.append( - { - "variant_id": variant_id, - "annotation_type": annotation_type, - "replace_all_versions": replace_all_versions, - "version": version, - } - ) + self._validate_subject(annotation_type, variant_id, allele_id) self._pending.append( - VariantAnnotationStatus( - variant_id=variant_id, + AnnotationEvent( annotation_type=annotation_type, - status=status, - version=version, - failure_category=failure_category, - current=current, + variant_id=variant_id, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + event_metadata=metadata, job_run_id=self.job_run_id, - **annotation_data, + score_set_id=score_set_id if score_set_id is not None else self.score_set_id, ) # type: ignore[call-arg] ) if len(self._pending) >= self.batch_size: self.flush() - def flush(self) -> None: - """Flush all pending annotations to the database. + @staticmethod + def _validate_subject(annotation_type: AnnotationType, variant_id: Optional[int], allele_id: Optional[int]) -> None: + if (variant_id is None) == (allele_id is None): + raise ValueError("Exactly one of variant_id or allele_id must be set") - Retires old ``current=True`` rows in bulk, then inserts all pending - new rows in a single ``add_all`` + ``flush``. This replaces the - previous pattern of 2 flushes per ``add_annotation`` call. - """ + type_value = annotation_type.value if isinstance(annotation_type, AnnotationType) else annotation_type + if variant_id is not None and type_value not in VARIANT_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is allele-subject; pass allele_id, not variant_id") + if allele_id is not None and type_value not in ALLELE_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is variant-subject; pass variant_id, not allele_id") + + def flush(self) -> None: + """Insert all pending events in a single ``add_all`` + ``flush``.""" if not self._pending: return - self._retire_existing() self.session.add_all(self._pending) self.session.flush() - logger.debug(f"Flushed {len(self._pending)} annotation statuses") + logger.debug(f"Flushed {len(self._pending)} variant events") self._pending.clear() - self._retirement_filters.clear() - - def _retire_existing(self) -> None: - """Bulk-retire existing current annotations for all pending writes. - - Groups retirement filters by (annotation_type, replace_all_versions, version) - and issues one UPDATE per group, minimizing round-trips. - """ - # Group filters to minimize UPDATE statements. - # Key: (annotation_type, replace_all_versions, version) -> list of variant_ids - groups: dict[tuple, list[int]] = {} - for f in self._retirement_filters: - key = (f["annotation_type"], f["replace_all_versions"], f["version"]) - groups.setdefault(key, []).append(f["variant_id"]) - - for (annotation_type, replace_all_versions, version), variant_ids in groups.items(): - conditions = [ - VariantAnnotationStatus.variant_id.in_(variant_ids), - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ] - if not replace_all_versions: - conditions.append(VariantAnnotationStatus.version == version) - - stmt = update(VariantAnnotationStatus).where(*conditions).values(current=False) - self.session.execute(stmt) def get_current_annotation( - self, variant_id: int, annotation_type: AnnotationType, version: Optional[str] = None - ) -> Optional[VariantAnnotationStatus]: - """ - Retrieve the current annotation for a given variant/type/version. - - Flushes pending annotations first to ensure the result is up to date. - """ - self.flush() - - stmt = select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) - - result = self.session.execute(stmt) - return result.scalar_one_or_none() - - def get_annotation_history( self, - variant_id: int, annotation_type: AnnotationType, - version: Optional[str] = None, - ) -> list[VariantAnnotationStatus]: - """ - Return the full annotation timeline for a variant/type, newest first. - - Includes both current and retired rows — useful for debugging and - support investigations. + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> Optional[AnnotationEvent]: + """Latest event for a single ``(subject, annotation_type)`` key. + + Current status is the newest event by ``id`` — there is no ``current`` + flag to filter on. Flushes pending events first so the result reflects + buffered writes. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - ) - .order_by(desc(VariantAnnotationStatus.id)) - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) - return list(self.session.scalars(stmt).all()) + stmt = stmt.order_by(desc(AnnotationEvent.id)).limit(1) + return self.session.scalars(stmt).first() - def get_all_current_annotations( + def get_event_history( self, - variant_id: int, - ) -> list[VariantAnnotationStatus]: - """ - Return all current annotations for a variant, across all types and versions. - - Useful for a quick overview of what annotations are active for a given variant. + annotation_type: AnnotationType, + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> list[AnnotationEvent]: + """Full event timeline for a ``(subject, annotation_type)`` key, newest first. + + The append-only log retains every observation — skips, reconfirms, + no-ops — so this is the complete audit trail, not just the current row. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.current.is_(True), - ) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.version) - ) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) + stmt = stmt.order_by(desc(AnnotationEvent.id)) return list(self.session.scalars(stmt).all()) diff --git a/src/mavedb/lib/authentication.py b/src/mavedb/lib/authentication.py index fed13ae1f..d10fad7ea 100644 --- a/src/mavedb/lib/authentication.py +++ b/src/mavedb/lib/authentication.py @@ -1,6 +1,6 @@ import logging import os -from datetime import datetime +from datetime import datetime, timedelta from enum import Enum from typing import Optional @@ -38,6 +38,10 @@ ACCESS_TOKEN_NAME = "X-API-key" +# How stale `last_login` may get before a request rewrites it. Nothing reads the value at request granularity, +# and writing it on every request costs a commit and a refresh on the hot path of every authenticated route. +LAST_LOGIN_REFRESH_INTERVAL = timedelta(minutes=15) + logger = logging.getLogger(__name__) @@ -121,7 +125,7 @@ async def get_access_token( return access_token_header or access_token_cookie -async def get_current_user_data_from_api_key( +def get_current_user_data_from_api_key( db: Session = Depends(deps.get_db), access_token: str = Depends(get_access_token) ) -> Optional[UserData]: user = None @@ -155,7 +159,7 @@ async def get_current_user_data_from_api_key( #################################################################################################### -async def get_current_user( +def get_current_user( api_key_user_data: Optional[UserData] = Depends(get_current_user_data_from_api_key), token_payload: Optional[dict] = Depends(JWTBearer()), db: Session = Depends(deps.get_db), @@ -205,6 +209,7 @@ async def get_current_user( email=email, is_first_login=True, ) + needs_write = True save_to_logging_context( { "user": user.id, @@ -220,8 +225,14 @@ async def get_current_user( logger.info(msg="Failed to authenticate user; User is inactive.", extra=logging_context()) return None else: - user.last_login = datetime.now() - user.is_first_login = False + needs_write = ( + user.is_first_login + or user.last_login is None + or datetime.now() - user.last_login >= LAST_LOGIN_REFRESH_INTERVAL + ) + if needs_write: + user.last_login = datetime.now() + user.is_first_login = False save_to_logging_context( { "user": user.id, @@ -230,17 +241,20 @@ async def get_current_user( } ) - db.add(user) - try: - db.commit() - except IntegrityError: + if needs_write: + db.add(user) + try: + db.commit() + # A concurrent request created this user between our initial query and this commit. # Roll back and re-fetch the existing record. - db.rollback() - user = db.query(User).filter(User.username == username).one() - logger.debug(msg="Concurrent first-login resolved; returning existing user.", extra=logging_context()) + except IntegrityError: + db.rollback() + user = db.query(User).filter(User.username == username).one() + logger.debug(msg="Concurrent first-login resolved; returning existing user.", extra=logging_context()) + + db.refresh(user) - db.refresh(user) logger.info(msg="Successfully authenticated user via JWT.", extra=logging_context()) # When no roles are requested, the user may act as any assigned role. diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py new file mode 100644 index 000000000..61dcaf5e4 --- /dev/null +++ b/src/mavedb/lib/cat_vrs.py @@ -0,0 +1,382 @@ +"""Cat-VRS transit, built on the fly from a variant's live allele links. + +The Categorical Variant served on ``/variants/{urn}`` is assembled per request from the variant's +live ``MappingRecordAllele`` links (see ``lib/alleles.py::get_live_record_allele_links``). +""" + +import logging +from dataclasses import dataclass +from enum import Enum +from typing import Mapping, Optional + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept +from ga4gh.cat_vrs.relations import ( + LIFTOVER_TO_RELATION, + TRANSCRIBED_TO_RELATION, + TRANSLATION_OF_RELATION, + Relation, +) +from ga4gh.core.models import Coding, ConceptMapping, iriReference +from ga4gh.core.models import Relation as MappingRelation +from ga4gh.vrs.models import Allele as VrsAllele +from ga4gh.vrs.models import CisPhasedBlock + +from mavedb.lib.allele_annotations import AlleleCrossReferences +from mavedb.lib.allele_identity import AlleleDerivation +from mavedb.lib.clingen.allele_registry import clingen_allele_url +from mavedb.lib.clinvar.utils import clinvar_variation_url +from mavedb.lib.gnomad import gnomad_variant_url +from mavedb.lib.logging.context import logging_context +from mavedb.lib.term_systems import ( + CLINGEN_ALLELE_REGISTRY, + CLINVAR_VARIATION, + GNOMAD, + MAVEDB_CAT_VRS_RELATION, +) +from mavedb.lib.term_systems import coding as term_coding +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.mapping_record_allele import MappingRecordAllele + +logger = logging.getLogger(__name__) + +# The spec's own concept for each Cat-VRS relation term. Each carries its term's system: Sequence Ontology +# for translation_of and transcribed_to, Cat-VRS's internal ga4gh-gkm-term vocabulary for liftover_to. +_SPEC_RELATION_CONCEPTS: dict[Relation, MappableConcept] = { + Relation.TRANSLATION_OF: TRANSLATION_OF_RELATION, + Relation.TRANSCRIBED_TO: TRANSCRIBED_TO_RELATION, + Relation.LIFTOVER_TO: LIFTOVER_TO_RELATION, +} + + +class CatVrsRelation(str, Enum): + """Relation of a member allele to the single defining (measured) allele. + + Deliberately **not** a subclass of Cat-VRS's ``Relation``: Python forbids extending an enum that + already has members, and inheriting would buy nothing anyway. Interop here is carried by + ``Coding.system`` on the emitted concept, not by Python enum identity — see + :func:`_relation_concept`, which maps the codes that have a spec equivalent onto it. Keeping this a + plain enum keeps the members statically visible to mypy and greppable. + """ + + # defining is protein; member is an nt allele (coding or genomic) that encodes it. Implied. + ENCODES = "encodes" + # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. + COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" + # defining is nt; member is the protein consequence. Consequence, no independent score. + TRANSLATION_OF = "translation_of" + # defining is nt; member is a *convergent encoding* — an nt allele not paired with the measured change that + # encodes the same protein consequence. + CO_ENCODES = "co_encodes" + + +# MaveDB code -> the Cat-VRS term it is an exact match for. A code with no entry has no spec equivalent. +_SPEC_EQUIVALENT: dict[CatVrsRelation, Relation] = { + CatVrsRelation.TRANSLATION_OF: Relation.TRANSLATION_OF, +} + + +class CatVrsMode(str, Enum): + """The score-collapse semantics of the categorical variant.""" + + PROJECTION = "projection" # Mode 1 — nt measured; score rides faithfully. + REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. + + @classmethod + def for_defining_level(cls, level: Optional[str]) -> "CatVrsMode": + """The mode a categorical variant takes from its defining (measured) allele's level.""" + return cls.REVERSE_TRANSLATION if level == SequenceLevel.protein.value else cls.PROJECTION + + +@dataclass +class CategoricalVariantTransit: + """The spec-pure Cat-VRS object plus the MaveDB layer that rides beside it.""" + + categorical_variant: CategoricalVariant + mode: CatVrsMode + + +def is_projection_partner(member_group: Optional[int], *, focus_group: Optional[int]) -> bool: + """Whether a member is the focus allele's own c↔g projection: the two share a projection group. + + A member or focus with no group can't be shown to pair, so it counts as no projection. That includes a + measured allele RT's fold-in missed (``fold_in_missed`` on its ``cross_level_translation`` event), whose + own twin then reads as a convergent encoding: under-claiming one exact match rather than presenting + distinct encodings, and their ClinVar/gnomAD records, as the measured change. + """ + return member_group is not None and member_group == focus_group + + +def member_label( + focus_level: Optional[str], member_level: Optional[str], *, is_projection: bool +) -> tuple[Optional[CatVrsRelation], Optional[AlleleDerivation]]: + """A member allele's ``(relation, derivation)`` relative to the focus allele, ``(None, None)`` for none. + + The single source of both axes for every view (Cat-VRS, variant detail, allele detail), so they cannot + label the same pair differently. ``is_projection`` is whether the member is the focus's own c↔g pair + (:func:`is_projection_partner` within a record). + + - Protein focus: every nucleotide member ``encodes`` it, a reverse-translation ``candidate``. + - Nucleotide focus: the protein consequence is its ``translation_of`` and its pair its + ``coordinate_representation_of``, both deterministic ``projection``s; any other nucleotide member + ``co_encodes`` the consequence, a distinct ``convergent`` change. + """ + if focus_level == SequenceLevel.protein.value: + if member_level in NUCLEOTIDE_LEVELS: + return CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE + return None, None + + if member_level == SequenceLevel.protein.value: + return CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION + if member_level in NUCLEOTIDE_LEVELS: + if is_projection: + return CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION + return CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT + + return None, None + + +def _relation_concept(relation: CatVrsRelation) -> MappableConcept: + """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept``. + + The MaveDB code is always the ``primaryCoding``. Codes are read *member -> defining*, which is + MaveDB's own framing, so claiming the spec's system for them outright would misrepresent their + provenance. Where a code *is* the same concept as a published Cat-VRS relation, an ``exactMatch`` + :class:`ConceptMapping` to that term is attached via ``MappableConcept.mappings``. + + A code with no spec equivalent carries no mapping. See :data:`_SPEC_EQUIVALENT` for which spec + gaps exist and why. + """ + spec_term = _SPEC_EQUIVALENT.get(relation) + return MappableConcept( + name=relation.value, + primaryCoding=term_coding(MAVEDB_CAT_VRS_RELATION, relation.value), + mappings=( + [ + ConceptMapping( + coding=_SPEC_RELATION_CONCEPTS[spec_term].primaryCoding, + relation=MappingRelation.EXACT_MATCH, + ) + ] + if spec_term is not None + else None + ), + ) + + +def _external_codings(allele: Allele, references: Optional[AlleleCrossReferences]) -> list[Coding]: + """The allele's ClinGen, gnomAD and ClinVar identifiers as resolvable codings. Identifiers only.""" + codings: list[Coding] = [] + if allele.clingen_allele_id: + caid = allele.clingen_allele_id + codings.append(term_coding(CLINGEN_ALLELE_REGISTRY, caid, iri=clingen_allele_url(caid))) + + if references is None: + return codings + + if references.gnomad is not None: + gnomad = references.gnomad + codings.append( + term_coding( + GNOMAD, + gnomad.db_identifier, + iri=gnomad_variant_url(gnomad.db_identifier, gnomad.db_version), + system_version=gnomad.db_version, + ) + ) + + for variation_id in references.clinvar_variation_ids: + codings.append(term_coding(CLINVAR_VARIATION, variation_id, iri=clinvar_variation_url(variation_id))) + + return codings + + +def _external_mappings( + members: list[tuple[Allele, Optional[CatVrsRelation]]], references: Mapping[str, AlleleCrossReferences] +) -> list[ConceptMapping]: + """Cross-reference the members' external records as ``CategoricalVariant.mappings``. + + ``members`` pairs each included allele with its relation to the defining allele, ``None`` marking the + defining allele itself. The defining allele and its coordinate representations are the categorical + variant, so their records are an ``exactMatch``. Every other member is a distinct variant tied to it by + encoding or translation, so its records are a ``relatedMatch``, following the ga4gh/cat-vrs v1 + protein-consequence examples (proteinSequenceConsequence-ex1.json). + + The c↔g twins of one change share a CAID, gnomAD ID and ClinVar variation, and a ClinVar variation + recurs once per live release, so mappings are deduplicated by (system, code) with ``exactMatch`` + taking precedence. Mappings are sorted by (system, code) so the output does not depend on the order + the database returned the links in. + """ + mappings: dict[tuple[str, str], ConceptMapping] = {} + for allele, relation in members: + is_exact = relation is None or relation is CatVrsRelation.COORDINATE_REPRESENTATION_OF + match = MappingRelation.EXACT_MATCH if is_exact else MappingRelation.RELATED_MATCH + + for coding in _external_codings(allele, references.get(allele.vrs_digest) if allele.vrs_digest else None): + key = (coding.system, coding.code.root) + existing = mappings.get(key) + if existing is None or (is_exact and existing.relation is not MappingRelation.EXACT_MATCH): + mappings[key] = ConceptMapping(coding=coding, relation=match) + + return [mappings[key] for key in sorted(mappings)] + + +def categorical_member_links( + links: list[MappingRecordAllele], *, include_convergent: bool = True +) -> list[MappingRecordAllele]: + """The links :func:`build_categorical_variant` draws members from, the defining (authoritative) link first. + + Empty when no link is authoritative. Selection is by level and projection group only; a selected + allele with no ``post_mapped`` is skipped later, at hydration. Callers use this to size or prefetch + for the categorical variant before building it. + """ + defining_link = next((link for link in links if link.is_authoritative), None) + if defining_link is None: + return [] + + # Projection mode, narrow object (include_convergent=False): drop any convergent encodings the + # reverse-translation fan left on the record, keeping only the measured change's precise coordinate + # projection and its protein consequence. + mode = CatVrsMode.for_defining_level(defining_link.allele.level) + members = [defining_link] + for link in links: + if link is defining_link: + continue + + relation, _ = member_label( + defining_link.allele.level, + link.allele.level, + is_projection=is_projection_partner(link.projection_group, focus_group=defining_link.projection_group), + ) + if mode is CatVrsMode.PROJECTION and not include_convergent and relation is CatVrsRelation.CO_ENCODES: + continue + + members.append(link) + + return members + + +def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: + """Bare VRS variation (Allele or CisPhasedBlock) from the stored post_mapped JSONB. + + ``None`` when the allele has no post_mapped representation — it cannot be a Cat-VRS member. + """ + if allele.post_mapped is None: + return None + + variation = vrs_object_from_mapped_variant(allele.post_mapped).root + # post_mapped only ever holds an Allele or CisPhasedBlock (the two shapes + # vrs_object_from_mapped_variant produces); MolecularVariation.root is a wider union, so narrow. + assert isinstance(variation, (VrsAllele, CisPhasedBlock)) + return variation + + +def build_categorical_variant( + links: list[MappingRecordAllele], + *, + name: str, + include_convergent: bool = True, + cross_references: Optional[Mapping[str, AlleleCrossReferences]] = None, +) -> Optional[CategoricalVariantTransit]: + """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. + + ``links`` is the record-scoped live link set from ``get_live_record_allele_links``. Exactly one + is authoritative (the measured/defining allele), the rest are derived members. Returns ``None`` + when there is no authoritative, hydratable link to anchor on (e.g. an unmapped variant). + + **The categorical variant always anchors on the measured allele**. Member selection therefore differs by mode: + + - **Reverse translation** (protein measured): the defining allele is the protein change and every + nt allele in the record ``encodes`` it, so the **full equivalence class** is unfurled. That class + is exactly what the protein measurement's claim ranges over. ``include_convergent`` is inert here. + - **Projection** (nt measured): the record also carries the reverse-translation fan out. This includes + all synonymous *convergent encodings* (other nt alleles encoding the same protein consequence, in + different projection groups), which are distinct, unmeasured variants rather than representations + of the measured change. ``include_convergent`` selects how they are handled: + + - ``True`` (default; the detail envelope): the encodings are kept as members wearing the + :attr:`CatVrsRelation.CO_ENCODES` relation, so the object is the full closure. The measured change's + projection and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. + - ``False`` (the VA-Spec subject): the encodings are dropped (see :func:`member_label`) and only the + measured change's precise projection and protein consequence remain. + + ``mappings`` cross-references the included members' external records (see :func:`_external_mappings`). + CAIDs come from the alleles themselves; gnomAD and ClinVar identifiers come from ``cross_references``, + the digest-keyed map from :func:`~mavedb.lib.allele_annotations.get_allele_cross_references` fetched at + the same ``as_of`` as ``links``. Without it, only CAIDs are mapped. + """ + member_links = categorical_member_links(links, include_convergent=include_convergent) + if not member_links: + return None + + defining_link, *other_links = member_links + defining_allele = defining_link.allele + defining_vrs = _hydrate_vrs(defining_allele) + + # The authoritative (measured) allele having no post_mapped breaks an invariant the mapping + # job upholds. Returning None here is analogous to "unmapped". + if defining_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: authoritative allele {defining_allele.vrs_digest!r} has no " + "post_mapped representation; treating the variant as unmapped." + ), + extra=logging_context(), + ) + return None + + defining_level = defining_allele.level + mode = CatVrsMode.for_defining_level(defining_level) + + members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] + mapped_members: list[tuple[Allele, Optional[CatVrsRelation]]] = [(defining_allele, None)] + relations_present: dict[CatVrsRelation, None] = {} # insertion-ordered set of relation kinds + + for link in other_links: + allele = link.allele + member_vrs = _hydrate_vrs(allele) + # A live link to an allele with no post_mapped is an unexpected data state. + if member_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: skipping member allele {allele.vrs_digest!r} with no " + "post_mapped representation." + ), + extra=logging_context(), + ) + continue + + members.append(member_vrs) + relation, _ = member_label( + defining_level, + allele.level, + is_projection=is_projection_partner(link.projection_group, focus_group=defining_link.projection_group), + ) + if relation is not None: + mapped_members.append((allele, relation)) + relations_present[relation] = None + + # The defining allele anchors the DefiningAlleleConstraint. Cat-VRS 1.0.0 requires a bare + # vrs:Allele (or an iri ref) there, so a multi-variant defining (CisPhasedBlock) is referenced by + # its digest instead. + defining_ref: VrsAllele | iriReference = ( + defining_vrs if isinstance(defining_vrs, VrsAllele) else iriReference(root=defining_allele.vrs_digest or "") + ) + + categorical_variant = CategoricalVariant( + name=name, + members=members, + constraints=[ + DefiningAlleleConstraint( + allele=defining_ref, + relations=[_relation_concept(relation) for relation in relations_present] or None, + ) + ], + mappings=_external_mappings(mapped_members, cross_references or {}) or None, + ) + + return CategoricalVariantTransit( + categorical_variant=categorical_variant, + mode=mode, + ) diff --git a/src/mavedb/lib/clingen/allele_registry.py b/src/mavedb/lib/clingen/allele_registry.py index b773e6891..d9e121bcb 100644 --- a/src/mavedb/lib/clingen/allele_registry.py +++ b/src/mavedb/lib/clingen/allele_registry.py @@ -1,16 +1,21 @@ import asyncio import logging from typing import Optional +from urllib.parse import quote import requests from aiocache import cached from mavedb.lib.clingen.cache import CACHE_CLASS, CACHE_CONFIG, CACHE_TTL_SECONDS, clingen_cache_key_builder +from mavedb.lib.clingen.constants import CLINGEN_ALLELE_REGISTRY_API_URL, CLINGEN_ALLELE_REGISTRY_PAGE_URL logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -CLINGEN_API_URL = "https://reg.genome.network/allele" + +def clingen_allele_url(clingen_allele_id: str) -> str: + """Allele Registry page for a CAID or PAID, in the form the ga4gh/cat-vrs examples use.""" + return f"{CLINGEN_ALLELE_REGISTRY_PAGE_URL}?caid={quote(clingen_allele_id, safe='')}" @cached(ttl=CACHE_TTL_SECONDS, key_builder=clingen_cache_key_builder, cache=CACHE_CLASS, **CACHE_CONFIG) @@ -33,7 +38,7 @@ async def get_clingen_allele_data(clingen_allele_id: str) -> Optional[dict]: (excluding 404, which returns None). """ loop = asyncio.get_running_loop() - response = await loop.run_in_executor(None, requests.get, f"{CLINGEN_API_URL}/{clingen_allele_id}") + response = await loop.run_in_executor(None, requests.get, f"{CLINGEN_ALLELE_REGISTRY_API_URL}/{clingen_allele_id}") if response.status_code == 404: return None @@ -44,6 +49,23 @@ async def get_clingen_allele_data(clingen_allele_id: str) -> Optional[dict]: return response.json() +async def refresh_clingen_allele_data_expiry(clingen_allele_id: str) -> bool: + """Restart the cache expiry of an allele's ClinGen data. + + A cache hit leaves the expiry set when the entry was written. Data a sibling score set cached + nearly a day earlier can therefore expire minutes after a warming step reads it, and the jobs + the warming step exists for find the cache cold. + + Args: + clingen_allele_id: ClinGen allele ID whose cached data to keep (e.g., CA123456). + + Returns: + True if the entry was cached and its expiry restarted, False if it was not cached. + """ + key = clingen_cache_key_builder(get_clingen_allele_data, clingen_allele_id) + return await get_clingen_allele_data.cache.expire(key, CACHE_TTL_SECONDS) # type: ignore[attr-defined] + + async def get_canonical_pa_ids(clingen_allele_id: str) -> list[str]: """Retrieve canonical PA IDs from the ClinGen API for a given ClinGen allele ID. @@ -136,80 +158,6 @@ async def get_associated_clinvar_allele_id(clingen_allele_id: str) -> str: return "" -def extract_hgvs_from_ca_allele_data( - data: dict, - target_is_coding: bool, - transcript_accession: Optional[str], -) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a CA (canonical allele) ID. - - Parses the ClinGen API response to find GRCh38 genomic HGVS, coding HGVS - matching the target transcript (or MANE fallback), and protein HGVS. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - target_is_coding: Whether the score set target is protein-coding. - transcript_accession: Specific transcript accession to match, or None to use MANE. - - Returns: - Tuple of (hgvs_g, hgvs_c, hgvs_p), any of which may be None. - """ - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - if data.get("genomicAlleles"): - for allele in data["genomicAlleles"]: - if allele.get("referenceGenome") == "GRCh38" and allele.get("hgvs"): - hgvs_g = allele["hgvs"][0] - break - - if target_is_coding and data.get("transcriptAlleles"): - if transcript_accession: - for allele in data["transcriptAlleles"]: - if allele.get("hgvs"): - for hgvs_string in allele["hgvs"]: - hgvs_reference_sequence = hgvs_string.split(":")[0] - if transcript_accession == hgvs_reference_sequence: - hgvs_c = hgvs_string - break - if hgvs_c: - if allele.get("proteinEffect"): - hgvs_p = allele["proteinEffect"].get("hgvs") - break - else: - # No transcript specified; use MANE if available - for allele in data["transcriptAlleles"]: - if allele.get("MANE"): - hgvs_c = allele["MANE"].get("nucleotide", {}).get("RefSeq", {}).get("hgvs") - hgvs_p = allele["MANE"].get("protein", {}).get("RefSeq", {}).get("hgvs") - break - - return hgvs_g, hgvs_c, hgvs_p - - -def extract_hgvs_from_pa_allele_data(data: dict) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a PA (protein allele) ID. - - For PA alleles, only hgvs_p is extracted from aminoAcidAlleles. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - - Returns: - Tuple of (None, None, hgvs_p), where hgvs_p may be None. - """ - hgvs_p: Optional[str] = None - - if data.get("aminoAcidAlleles"): - for allele in data["aminoAcidAlleles"]: - if allele.get("hgvs"): - hgvs_p = allele["hgvs"][0] - break - - return None, None, hgvs_p - - def expand_allele_ids(clingen_allele_ids: list[Optional[str]]) -> set[str]: """Expand comma-separated multi-variant ClinGen allele IDs into individual IDs. diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py new file mode 100644 index 000000000..562dab94e --- /dev/null +++ b/src/mavedb/lib/clingen/alleles.py @@ -0,0 +1,111 @@ +"""Query helpers for fetching score-set alleles for ClinGen registration. + +Both submit_score_set_mappings_to_car and warm_clingen_cache use the same allele +scope: all current MappingRecordAllele links (authoritative and RT-derived) for a +score set. A single definition here prevents the two jobs from drifting apart. +""" + +from typing import Callable, Iterable, NamedTuple, Optional, TypeVar + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + +P = TypeVar("P") + + +class ScoreSetAlleleRow(NamedTuple): + """One (allele, variant) link for a score set. An allele shared by multiple variants appears once + per variant; :func:`group_alleles_for_annotation` collapses those duplicates into one work-unit + per allele. + + ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the + VEP job can build its HGVS payload without a second query. ``level`` is the allele's sequence + level (``protein``/``cdna``/``genomic``), carried so jobs annotating just nucleotide alleles can + skip protein alleles without a second query. All are optional with a ``None`` default so payloads keying + only on the CAID need not name them; real DB rows always populate them. + """ + + allele_id: int + post_mapped: dict | None + clingen_allele_id: str | None + variant_id: int + hgvs_g: str | None = None + hgvs_c: str | None = None + hgvs_p: str | None = None + level: str | None = None + + +def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: + """Return all current alleles for a score set with their linked variant IDs. + + Covers both authoritative mapper alleles and RT-derived equivalence alleles — + the full set that requires ClinGen registration before the annotation fan-out + can run. + + Only alleles with a non-null ``post_mapped`` are returned — variants that failed + or were benignly absent have no allele link and cannot receive a CAID. + """ + rows = db.execute( + select( + Allele.id, + Allele.post_mapped, + Allele.clingen_allele_id, + Variant.id.label("variant_id"), + Allele.hgvs_g, + Allele.hgvs_c, + Allele.hgvs_p, + Allele.level, + ) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(Allele.post_mapped.is_not(None)) + ).all() + + return [ + ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p, r.level) + for r in rows + ] + + +def group_alleles_for_annotation( + rows: Iterable[ScoreSetAlleleRow], + payload: Callable[[ScoreSetAlleleRow], Optional[P]], +) -> dict[int, P]: + """Collapse the per-(allele, variant) rows from :func:`get_alleles_for_score_set` into one + job-specific payload per allele, keyed by ``allele_id``. + + The same allele recurs once per variant that links it, so this dedups those rows down to one + entry per allele — the shape every allele-keyed annotation job wants now that annotation events + are allele-keyed (one event per allele, never fanned per-variant). + + ``payload`` builds the job-specific payload from the first row seen for an allele — the CAID for + gnomAD/ClinVar, the HGVS for VEP, etc. Returning ``None`` skips the allele entirely (e.g. it + carries no CAID), replacing each job's ad-hoc ``if row.x is None: continue``. ``payload`` must be + a pure function of allele-level fields so its result is stable across an allele's rows. + + Grouping on ``allele_id`` rather than ``vrs_digest`` is intentional: the two are 1:1 (content + addressing makes ``vrs_digest`` unique), so the groups are identical either way, and ``allele_id`` + is the permanent, never-reused surrogate. If annotation storage later keys on the digest, carry + it on the row and store against it — the grouping contract here does not change. + """ + groups: dict[int, P] = {} + for row in rows: + if row.allele_id in groups: + continue + + built = payload(row) + if built is None: + continue + + groups[row.allele_id] = built + + return groups diff --git a/src/mavedb/lib/clingen/constants.py b/src/mavedb/lib/clingen/constants.py index 2276ee187..cf0f7c56c 100644 --- a/src/mavedb/lib/clingen/constants.py +++ b/src/mavedb/lib/clingen/constants.py @@ -9,13 +9,29 @@ CAR_SUBMISSION_ENDPOINT = os.getenv("CAR_SUBMISSION_ENDPOINT") +CLINGEN_ALLELE_REGISTRY_API_URL = "https://reg.genome.network/allele" +"""ClinGen Allele Registry REST endpoint for looking up an allele by CAID, PAID or HGVS.""" +CLINGEN_ALLELE_REGISTRY_PAGE_URL = "https://reg.clinicalgenome.org/redmine/projects/registry/genboree_registry/by_caid" +"""Allele Registry web page for a CAID or PAID, selected with a ``caid`` query parameter.""" +GENBOREE_AUTH_URL = "https://genboree.org/auth/usr" +"""Genboree account authentication, which issues the JWT for ClinGen submissions.""" + LDH_SUBMISSION_TYPE = "cg-ldh-ld-submission" LDH_ENTITY_NAME = "MaveDBMapping" LDH_ENTITY_ENDPOINT = "maveDb" # for some reason, not the same :/ DEFAULT_LDH_SUBMISSION_BATCH_SIZE = 100 +DEFAULT_CAR_SUBMISSION_BATCH_SIZE = int(os.getenv("CAR_SUBMISSION_BATCH_SIZE", "10000")) +"""Number of HGVS strings sent to the ClinGen Allele Registry per PUT. Reverse translation can +produce very large allele counts, so submissions are chunked rather than sent as a single payload.""" CLINGEN_CACHE_WARMING_CONCURRENCY = int(os.getenv("CLINGEN_CACHE_WARMING_CONCURRENCY", "5")) """Maximum number of concurrent requests to make to the ClinGen API when pre-warming the cache for mapped variants.""" + +CLINGEN_CONNECT_TIMEOUT = float(os.getenv("CLINGEN_CONNECT_TIMEOUT", "5")) +CLINGEN_READ_TIMEOUT = float(os.getenv("CLINGEN_READ_TIMEOUT", "300")) +CLINGEN_HTTP_TIMEOUT = (CLINGEN_CONNECT_TIMEOUT, CLINGEN_READ_TIMEOUT) +"""(connect, read) timeout for ClinGen HTTP calls. Fail fast on connect, with a generous read +budget because a large batch can take CAR a while to respond.""" LDH_SUBMISSION_ENDPOINT = f"https://genboree.org/mq/brdg/pulsar/{CLIN_GEN_TENANT}/ldh/submissions/{LDH_ENTITY_ENDPOINT}" LDH_ACCESS_ENDPOINT = os.getenv("LDH_ACCESS_ENDPOINT", "https://genboree.org/ldh") LDH_MAVE_ACCESS_ENDPOINT = f"{LDH_ACCESS_ENDPOINT}/{LDH_ENTITY_NAME}/id" diff --git a/src/mavedb/lib/clingen/content_constructors.py b/src/mavedb/lib/clingen/content_constructors.py index 1f55437f4..ce62ebbbd 100644 --- a/src/mavedb/lib/clingen/content_constructors.py +++ b/src/mavedb/lib/clingen/content_constructors.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import Optional from uuid import uuid4 from urllib.parse import quote_plus @@ -7,7 +6,8 @@ from mavedb.constants import MAVEDB_BASE_GIT, MAVEDB_FRONTEND_URL from mavedb.lib.types.clingen import LdhContentLinkedData, LdhContentSubject, LdhEvent, LdhSubmission from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord from mavedb.models.variant import Variant @@ -32,8 +32,12 @@ def construct_ldh_submission_subject(hgvs: str) -> LdhContentSubject: return {"Variant": {"hgvs": hgvs}} -def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[MappedVariant]) -> LdhContentLinkedData: - entity: LdhContentLinkedData = { +def construct_ldh_submission_entity( + variant: Variant, mapping_record: MappingRecord, allele: Allele +) -> LdhContentLinkedData: + # Pre-mapped data and the mapping API version live on the per-variant MappingRecord; + # post-mapped data lives on the (cross-variant deduped) Allele. + return { # TODO#372: We try to make all possible fields that are non-nullable represented that way. "MaveDBMapping": [ { @@ -41,27 +45,25 @@ def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[M "mavedb_id": variant.urn, # type: ignore "score": variant.data["score_data"]["score"], # type: ignore "score_set_description": variant.score_set.short_description, # type: ignore + "pre_mapped": mapping_record.pre_mapped, + "post_mapped": allele.post_mapped, + "mapping_api_version": mapping_record.mapping_api_version, }, "entId": variant.urn, # type: ignore "entIri": f"{MAVEDB_FRONTEND_URL}/score-sets/{quote_plus(variant.score_set.urn)}?variant={quote_plus(variant.urn)}", # type: ignore } ] } - if mapped_variant is not None: - entity["MaveDBMapping"][0]["entContent"]["pre_mapped"] = mapped_variant.pre_mapped - entity["MaveDBMapping"][0]["entContent"]["post_mapped"] = mapped_variant.post_mapped - entity["MaveDBMapping"][0]["entContent"]["mapping_api_version"] = mapped_variant.mapping_api_version - return entity def construct_ldh_submission( - variant_content: list[tuple[str, Variant, Optional[MappedVariant]]], + variant_content: list[tuple[str, Variant, MappingRecord, Allele]], ) -> list[LdhSubmission]: content_submission: list[LdhSubmission] = [] - for hgvs, variant, mapped_variant in variant_content: + for hgvs, variant, mapping_record, allele in variant_content: subject = construct_ldh_submission_subject(hgvs) event = construct_ldh_submission_event(subject) - entity = construct_ldh_submission_entity(variant, mapped_variant) + entity = construct_ldh_submission_entity(variant, mapping_record, allele) content_submission.append( { diff --git a/src/mavedb/lib/clingen/services.py b/src/mavedb/lib/clingen/services.py index 085d9103f..fb491f825 100644 --- a/src/mavedb/lib/clingen/services.py +++ b/src/mavedb/lib/clingen/services.py @@ -8,7 +8,12 @@ import requests from jose import jwt -from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD +from mavedb.lib.clingen.constants import ( + CLINGEN_HTTP_TIMEOUT, + GENBOREE_ACCOUNT_NAME, + GENBOREE_ACCOUNT_PASSWORD, + GENBOREE_AUTH_URL, +) from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context from mavedb.lib.types.clingen import ClinGenAllele, ClinGenSubmissionError, LdhSubmission from mavedb.lib.utils import batched @@ -80,6 +85,7 @@ def dispatch_submissions( response = requests.put( url=request_url, data="\n".join(content_submissions), + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() @@ -163,12 +169,12 @@ def authenticate(self) -> str: extra=logging_context(), ) - auth_url = f"https://genboree.org/auth/usr/gb:{GENBOREE_ACCOUNT_NAME}/auth" + auth_url = f"{GENBOREE_AUTH_URL}/gb:{GENBOREE_ACCOUNT_NAME}/auth" auth_body = {"type": "plain", "val": GENBOREE_ACCOUNT_PASSWORD} - auth_response = requests.post(auth_url, json=auth_body) try: + auth_response = requests.post(auth_url, json=auth_body, timeout=CLINGEN_HTTP_TIMEOUT) auth_response.raise_for_status() - except requests.exceptions.HTTPError as exc: + except requests.exceptions.RequestException as exc: save_to_logging_context(format_raised_exception_info_as_dict(exc)) logger.error(msg="Failed to authenticate with Genboree services.", exc_info=exc, extra=logging_context()) raise exc @@ -221,16 +227,17 @@ def dispatch_submissions( logger.info(msg=f"Dispatching {len(submissions)} ldh submissions...", extra=logging_context()) for idx, content in enumerate(submissions): try: - logger.debug(msg=f"Dispatching submission {idx+1}.", extra=logging_context()) + logger.debug(msg=f"Dispatching submission {idx + 1}.", extra=logging_context()) response = requests.put( url=self.url, json=content, headers={"Authorization": f"Bearer {self.authenticate()}", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() submission_successes.append(response.json()) logger.info( - msg=f"Successfully dispatched ldh submission ({idx+1} / {len(submissions)}).", + msg=f"Successfully dispatched ldh submission ({idx + 1} / {len(submissions)}).", extra=logging_context(), ) diff --git a/src/mavedb/lib/clinical_controls.py b/src/mavedb/lib/clinical_controls.py new file mode 100644 index 000000000..3f9fd41e0 --- /dev/null +++ b/src/mavedb/lib/clinical_controls.py @@ -0,0 +1,126 @@ +"""Live ClinVar-control lookups for a score set, over the allele-link substrate. + +A score set's clinical controls are the ClinVar assertions whose allele is live-linked to one of the +score set's variants, reached by walking ``ClinvarAlleleLink → Allele → MappingRecordAllele → +MappingRecord → Variant`` with every ``ValidTime`` hop constrained to the same instant (``as_of``, +defaulting to currently-live). Both serving endpoints — the controls list +(``GET /score-sets/{urn}/clinical-controls``) and its options facet (``.../options``) walk the chain +that is defined here. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional, TypeVar + +from sqlalchemy import Select, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + +SelectT = TypeVar("SelectT", bound=Select[Any]) + + +@dataclass(frozen=True) +class ControlVariantLink: + """One (score-set variant, annotated allele) pair that a clinical control reaches. + + ``allele_digest`` is the VRS digest of the allele the control annotates. A client compares it to + the variant's authoritative ``assay_level_digest``: + - a match is a call *on the assayed level* (this call takes precedence) + - a mismatch is a call on an *encoding* level (this call is only used when the assayed level is + unannotated). + + Without the digest, the two are indistinguishable and precedence is unimplementable. + """ + + variant_urn: str + allele_digest: str + + +def _scope_to_live_score_set_controls(stmt: SelectT, score_set_id: int, as_of: Optional[datetime]) -> SelectT: + """Join ``stmt`` (already selecting from :class:`ClinvarControl`) down the allele-link chain to the + score set's variants and constrain every ``ValidTime`` hop to ``as_of``. ``distinct`` because the + chain fans a control out across each variant it links to.""" + return ( + stmt.join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + + +def get_clinical_controls_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_name: Optional[str] = None, + db_version: Optional[str] = None, +) -> list[tuple[ClinvarControl, list[ControlVariantLink]]]: + """The live ClinVar controls for a score set, each paired with the score-set variants that link to + it. Every link carrying the digest of the allele the control annotates (see :class:`ControlVariantLink`). + Optionally narrowed to one control DB (``db_name``) and/or release (``db_version``). Controls come back + in first-seen order. Each control's links preserve query order.""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl, Variant.urn.label("variant_urn"), Allele.vrs_digest.label("allele_digest")), + score_set_id, + as_of, + ) + if db_name is not None: + stmt = stmt.where(ClinvarControl.db_name == db_name) + if db_version is not None: + stmt = stmt.where(ClinvarControl.db_version == db_version) + + controls: dict[int, ClinvarControl] = {} + links_by_control: dict[int, list[ControlVariantLink]] = {} + + # A persisted variant reached through the join always carries a URN. Narrow the nullable column so the + # URN matches the view model's required `str`. TODO(#372) + for ctrl, variant_urn, allele_digest in db.execute(stmt).tuples(): + if variant_urn is None: + continue + + if ctrl.id not in controls: + controls[ctrl.id] = ctrl + links_by_control[ctrl.id] = [] + links_by_control[ctrl.id].append(ControlVariantLink(variant_urn=variant_urn, allele_digest=allele_digest)) + + return [(ctrl, links_by_control[ctrl.id]) for ctrl in controls.values()] + + +def get_clinical_control_options( + db: Session, score_set_id: int, *, as_of: Optional[datetime] = None +) -> list[tuple[str, list[str]]]: + """The options facet: each control DB live-linked to the score set, paired with its available + versions newest-first.""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl.db_name, ClinvarControl.db_version), score_set_id, as_of + ) + options: dict[str, list[str]] = {} + for db_name, db_version in db.execute(stmt): + options.setdefault(db_name, []).append(db_version) + + return [ + (db_name, sorted(versions, key=clinvar_version_sort_key, reverse=True)) for db_name, versions in options.items() + ] + + +def clinvar_version_sort_key(version: str) -> tuple[int, int]: + """Sort key for a ClinVar release string in ``MM_YYYY`` form, ordering by ``(year, month)``. + Unparseable versions sort to the bottom as ``(0, 0)`` rather than raising.""" + try: + month, year = version.split("_") + return int(year), int(month) + except (ValueError, AttributeError): + return (0, 0) diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index e70c4fee2..11f028fc8 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -4,6 +4,8 @@ from urllib3.util.retry import Retry TSV_VARIANT_ARCHIVE_BASE_URL = "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/archive" +CLINVAR_VARIATION_URL = "https://www.ncbi.nlm.nih.gov/clinvar/variation" +"""ClinVar web page for a variation, addressed as ``{CLINVAR_VARIATION_URL}/{variation_id}/``.""" NCBI_REQUEST_HEADERS = { "User-Agent": "MaveDB/1.0 (https://mavedb.org)", @@ -26,6 +28,10 @@ backoff for throttling is unnecessary — a modest retry with short backoff suffices. """ -CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus") +CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus", "VariationID") """Only these fields are extracted from each ClinVar TSV row and cached. The full TSV has ~30 columns; trimming to only what we need shrinks the cached pickle from hundreds of MB to tens of MB and speeds up load times. + +VariationID is ClinVar's canonical public identifier (anchors the web UI / variation links); we keep it +alongside the AlleleID (the row key) so the link record can carry both. A row missing the column on an +older archival TSV degrades to None rather than failing the whole version's parse. """ diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index 689e369ea..7ac13e5d1 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -9,6 +9,7 @@ from datetime import datetime from pathlib import Path from typing import Dict +from urllib.parse import quote import requests from filelock import FileLock @@ -17,6 +18,7 @@ from mavedb.lib.clinvar.constants import ( CLINVAR_CACHE_DIR, CLINVAR_FIELDS_TO_KEEP, + CLINVAR_VARIATION_URL, NCBI_REQUEST_HEADERS, NCBI_RETRY_STRATEGY, TSV_VARIANT_ARCHIVE_BASE_URL, @@ -28,6 +30,11 @@ logger = logging.getLogger(__name__) +def clinvar_variation_url(variation_id: str) -> str: + """Link to a ClinVar variation record. Mirrors ``clinvarVariationUrl`` in the UI's ``src/lib/clinvar-controls.ts``.""" + return f"{CLINVAR_VARIATION_URL}/{quote(variation_id, safe='')}/" + + def _ncbi_session() -> requests.Session: session = requests.Session() session.headers.update(NCBI_REQUEST_HEADERS) @@ -175,8 +182,11 @@ def _fetch_parse_and_cache( # as a list (which would be 1.5–2 GB for a modern TSV). with gzip.open(filename=buf, mode="rt") as f: reader = csv.DictReader(f, delimiter="\t") # type: ignore + # row.get (not row[field]) so a field absent from an older archival TSV schema yields + # None for that row rather than raising and discarding the whole version's parse. data: Dict[str, Dict[str, str]] = { - str(row["#AlleleID"]): {field: row[field] for field in CLINVAR_FIELDS_TO_KEEP} for row in reader + str(row["#AlleleID"]): {field: row.get(field) for field in CLINVAR_FIELDS_TO_KEEP} # type: ignore[misc] + for row in reader } finally: csv.field_size_limit(default_csv_field_size_limit) diff --git a/src/mavedb/lib/collections.py b/src/mavedb/lib/collections.py new file mode 100644 index 000000000..3e57bae28 --- /dev/null +++ b/src/mavedb/lib/collections.py @@ -0,0 +1,45 @@ +from typing import Iterable, Optional + +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_sets import readable_score_set_urns +from mavedb.lib.types.authentication import UserData +from mavedb.models.collection import Collection +from mavedb.view_models.collection import OfficialCollection + + +def readable_official_collections( + user_data: Optional[UserData], collections: Iterable[Collection] +) -> list[OfficialCollection]: + """Serialize official collections, carrying only what the caller may read. + + An official collection is a badge on every score set and experiment it holds, so it is serialized into + responses about those records. Collections the caller may not read are dropped, and each remaining + collection's member URNs are narrowed to the members the caller may read: a public score set can share + a badge collection with private ones. + + Narrows validated views rather than the ORM collections, whose member relationships would stage writes + if reassigned. + """ + narrowed = [] + for collection in collections: + if not has_permission(user_data, collection, Action.READ).permitted: + continue + + score_set_urns = readable_score_set_urns(user_data, collection.score_sets) + experiment_urns = { + experiment.urn + for experiment in collection.experiments + if has_permission(user_data, experiment, Action.READ).permitted + } + + view = OfficialCollection.model_validate(collection) + narrowed.append( + view.model_copy( + update={ + "score_set_urns": [urn for urn in view.score_set_urns if urn in score_set_urns], + "experiment_urns": [urn for urn in view.experiment_urns if urn in experiment_urns], + } + ) + ) + + return narrowed diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py index 3469d6d64..c037041b8 100644 --- a/src/mavedb/lib/csv/annotations.py +++ b/src/mavedb/lib/csv/annotations.py @@ -10,8 +10,8 @@ from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation from mavedb.lib.csv.entries import calibration_viewer +from mavedb.lib.csv.specs import CsvMappedRow from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_calibration_functional_classification_variant_association import ( @@ -74,7 +74,7 @@ def containing_classification_ids(db: Session, variant_ids: Sequence[int]) -> di def annotations_for_rows( db: Session, variants: Sequence[Variant], - mappings: Sequence[Optional[MappedVariant]], + mappings: Sequence[Optional[CsvMappedRow]], calibration_namespaces: dict[str, str], viewer: Optional[ScoreCalibrationViewer] = None, ) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: @@ -103,7 +103,7 @@ def annotations_for_rows( if mapping is None or calibration is None or calibration.score_set_id != variant.score_set_id: annotations[namespace] = None else: - annotations[namespace] = flatten_annotation(mapping, calibration, contained) + annotations[namespace] = flatten_annotation(variant, calibration, contained) rows.append(annotations) return rows diff --git a/src/mavedb/lib/csv/build_limiter.py b/src/mavedb/lib/csv/build_limiter.py new file mode 100644 index 000000000..a3f8b43cb --- /dev/null +++ b/src/mavedb/lib/csv/build_limiter.py @@ -0,0 +1,67 @@ +"""Load shedding for the score set CSV exports. + +Building a large CSV is CPU-bound Python, and Python runs one thread at a time per process. A few large builds +running at once in one API process starve everything else that process serves, so each process runs only a few at a +time and refuses the rest with a 503 the client can retry. + +TODO(#845, #771): A stopgap. Remove it once full exports are cheap or served from cache, and ``csv_build_refused`` +no longer appears in the request logs. +""" + +import logging +import os +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Optional + +from fastapi import HTTPException + +from mavedb.constants import MAVEDB_BULK_DOWNLOAD_URL +from mavedb.lib.logging.context import logging_context, save_to_logging_context + +logger = logging.getLogger(__name__) + +# Large builds one API process runs at once; 0 disables the cap. +CSV_BUILD_SLOTS = int(os.getenv("CSV_BUILD_SLOTS", "2")) +# Builds of fewer rows skip the cap, so previews and small score sets are never refused. +CSV_BUILD_MIN_ROWS = int(os.getenv("CSV_BUILD_MIN_ROWS", "10000")) +# How long a request waits for a slot. Leaves most of the load balancer's 60s idle timeout for the build itself. +CSV_BUILD_WAIT_SECONDS = 10 +RETRY_AFTER_SECONDS = 30 + +BUSY_DETAIL = ( + "The server is busy building other large downloads. Retry after the number of seconds in the Retry-After header." + f" To download many datasets, consider using the bulk archive instead: {MAVEDB_BULK_DOWNLOAD_URL}" +) + +_slots = threading.BoundedSemaphore(CSV_BUILD_SLOTS) if CSV_BUILD_SLOTS else None + + +def rows_to_build(num_variants: Optional[int], start: Optional[int], limit: Optional[int]) -> int: + """The number of rows a request for ``start``/``limit`` of a score set with ``num_variants`` variants returns.""" + # TODO(#372): `num_variants` is NOT NULL but typed optional. + remaining = max((num_variants or 0) - (start or 0), 0) + return min(remaining, limit) if limit else remaining + + +@contextmanager +def csv_build_slot(rows: int) -> Iterator[None]: + """Hold one of this process's CSV build slots while building ``rows`` rows. + + Waits up to ``CSV_BUILD_WAIT_SECONDS`` for a slot, then raises a 503 with ``Retry-After``. Builds smaller than + ``CSV_BUILD_MIN_ROWS`` run without a slot. + """ + if _slots is None or rows < CSV_BUILD_MIN_ROWS: + yield + return + + if not _slots.acquire(timeout=CSV_BUILD_WAIT_SECONDS): + save_to_logging_context({"csv_build_refused": True, "csv_build_rows": rows}) + logger.info(msg="Refused a large CSV build; every build slot is in use.", extra=logging_context()) + raise HTTPException(status_code=503, detail=BUSY_DETAIL, headers={"Retry-After": str(RETRY_AFTER_SECONDS)}) + + try: + yield + finally: + _slots.release() diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py index a086aea71..2634c14b3 100644 --- a/src/mavedb/lib/csv/columns.py +++ b/src/mavedb/lib/csv/columns.py @@ -16,12 +16,11 @@ parse_calibration_namespace, parse_clinvar_namespace, ) -from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from mavedb.lib.csv.specs import CORE_NAMESPACE, CsvMappedRow, RowSource, namespace_spec from mavedb.lib.mave.utils import NA_VALUE, NULL_VALUES from mavedb.lib.validation.constants.general import hgvs_columns -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant _OUTPUT_NULL_STRINGS = frozenset(value.lower() for value in NULL_VALUES if value) @@ -133,9 +132,9 @@ def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: b def variant_to_csv_row( variant: Variant, columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, + mapping: Optional[CsvMappedRow] = None, gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinvarControl]]] = None, annotations_by_ns: Optional[dict[str, Optional[FlatAnnotation]]] = None, match_type: Optional[str] = None, namespaced: bool = False, @@ -189,9 +188,9 @@ def variant_to_csv_row( def variants_to_csv_rows( variants: Sequence[Variant], columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + mappings: Optional[Sequence[Optional[CsvMappedRow]]] = None, gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinvarControl]]]]] = None, annotations_by_ns: Optional[Sequence[Optional[dict[str, Optional[FlatAnnotation]]]]] = None, match_types: Optional[Sequence[Optional[str]]] = None, namespaced: bool = False, @@ -199,9 +198,9 @@ def variants_to_csv_rows( ) -> Iterable[dict[str, Any]]: """Format each variant into a dictionary row containing the keys specified in *columns*.""" n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _mappings: Sequence[Optional[CsvMappedRow]] = mappings if mappings is not None else [None] * n _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + _clinvar: Sequence[Optional[dict[str, Optional[ClinvarControl]]]] = ( clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n ) _annotations: Sequence[Optional[dict[str, Optional[FlatAnnotation]]]] = ( diff --git a/src/mavedb/lib/csv/deprecated_params.py b/src/mavedb/lib/csv/deprecated_params.py index 3199fce5d..e4c692f1d 100644 --- a/src/mavedb/lib/csv/deprecated_params.py +++ b/src/mavedb/lib/csv/deprecated_params.py @@ -10,14 +10,11 @@ TODO(#XXX): remove this module once clients have migrated. """ -import logging from dataclasses import dataclass, field from typing import List, Optional from mavedb.lib.csv.namespaces import CsvNamespace -from mavedb.lib.logging.context import save_to_logging_context - -logger = logging.getLogger(__name__) +from mavedb.lib.deprecation import deprecation_headers, record_deprecated_usage DROP_NA_COLUMNS_DESCRIPTION = ( @@ -48,12 +45,7 @@ class ResolvedCsvParams: def _record(self, name: str, replacement: str) -> None: self.deprecations[name] = replacement - save_to_logging_context({"deprecated_query_parameters": sorted(self.deprecations), "deprecation_marker": True}) - logger.warning( - msg=f"Request used the deprecated query parameter '{name}'; it will be removed in a future" - f" release. Use '{replacement}' instead.", - extra={"deprecated_query_parameter": name, "replacement_query_parameter": replacement}, - ) + record_deprecated_usage(name, successor=replacement) @property def response_headers(self) -> dict[str, str]: @@ -64,11 +56,7 @@ def response_headers(self) -> dict[str, str]: warnings = "; ".join( f"{name} is deprecated, use {replacement}" for name, replacement in sorted(self.deprecations.items()) ) - return { - # RFC 8594. No Sunset header: the removal release is not scheduled. - "Deprecation": "true", - "Warning": f'299 - "{warnings}"', - } + return deprecation_headers(successor=None, warning=warnings) def resolve_deprecated_csv_params( diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py index ba38b7f60..ed46c9ec6 100644 --- a/src/mavedb/lib/csv/entries.py +++ b/src/mavedb/lib/csv/entries.py @@ -6,12 +6,13 @@ """ from dataclasses import dataclass +from datetime import datetime from typing import Iterable, Optional, Sequence from sqlalchemy import and_, select from sqlalchemy.orm import Session -from mavedb.lib.annotation.util import score_calibration_may_be_used_for_annotation +from mavedb.lib.annotation.calibration import score_calibration_may_be_used_for_annotation from mavedb.lib.csv.namespaces import ( CLINVAR_DB_NAME, STATIC_CSV_NAMESPACE_LABELS, @@ -22,8 +23,10 @@ clinvar_namespace_sort_key, ) from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -118,7 +121,7 @@ def calibration_can_annotate(calibration: ScoreCalibration) -> bool: def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> list[AvailableCsvNamespaceEntry]: """Build labeled entries for calibrations, named by title so a picker can identify them. - Research-use-only calibrations (labelled with a prefix) and rangeless ones are offered but excluded + Research-use-only calibrations (flagged for a picker to tag) and rangeless ones are offered but excluded from the default selection. """ entries = [] @@ -131,7 +134,7 @@ def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> l entries.append( AvailableCsvNamespaceEntry( namespace=calibration_namespace_for_urn(str(calibration.urn)), - label=f"Research Use Only: {title}" if research_use_only else title, + label=title, group=CsvNamespaceGroup.CALIBRATION, score_set=calibration.score_set, research_use_only=research_use_only, @@ -142,8 +145,10 @@ def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> l return entries -def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) -> bool: - """Whether any variant in these score sets has a current mapping. +def score_sets_have_current_mappings( + db: Session, score_set_ids: Sequence[int], *, as_of: Optional[datetime] = None +) -> bool: + """Whether any variant in these score sets has a live mapping record with an authoritative allele. Gates the mapping-derived namespaces: any mapping in a score set means the variant CSV should offer the namespaces, even if the variant in question is unmapped. @@ -153,16 +158,25 @@ def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) return ( db.scalars( - select(MappedVariant.id) - .join(MappedVariant.variant) - .where(and_(Variant.score_set_id.in_(score_set_ids), MappedVariant.current.is_(True))) + select(MappingRecordAllele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where( + and_( + Variant.score_set_id.in_(score_set_ids), + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ) + ) .limit(1) ).first() is not None ) -def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> list[str]: +def clinvar_release_namespaces( + db: Session, score_set_ids: Sequence[int], *, as_of: Optional[datetime] = None +) -> list[str]: """Every ClinVar release namespace these score sets have data for. Scoped to the score set, not the measurement, so a variant with no record still gets NA columns — @@ -173,14 +187,18 @@ def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> lis return [] db_versions = db.scalars( - select(ClinicalControl.db_version) - .join(ClinicalControl.mapped_variants.of_type(MappedVariant)) - .join(MappedVariant.variant) + select(ClinvarControl.db_version) + .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == ClinvarAlleleLink.allele_id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) .where( and_( Variant.score_set_id.in_(score_set_ids), - MappedVariant.current.is_(True), - ClinicalControl.db_name == CLINVAR_DB_NAME, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ClinvarAlleleLink.live_at(as_of), + ClinvarControl.db_name == CLINVAR_DB_NAME, ) ) .distinct() diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py index 9244db15c..60d5f2f7f 100644 --- a/src/mavedb/lib/csv/fetch.py +++ b/src/mavedb/lib/csv/fetch.py @@ -5,28 +5,177 @@ """ from dataclasses import dataclass +from datetime import datetime from typing import Any, Optional, Sequence -from sqlalchemy import Integer, and_, cast, func, select -from sqlalchemy.orm import Session, aliased, selectinload +from sqlalchemy import Select, and_, select +from sqlalchemy.orm import Session, selectinload from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME -from mavedb.lib.csv.specs import namespace_spec -from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.lib.csv.specs import CsvMappedRow, namespace_spec +from mavedb.lib.gnomad import GNOMAD_DB_NAME +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +# CSV rows come out in variant-number order (the integer after '#'); id breaks ties stably. +_VARIANT_NUMBER_ORDER = (Variant.variant_number, Variant.id) @dataclass class CsvFetchResult: variants: list[Variant] - mappings: Optional[list[Optional[MappedVariant]]] + mappings: Optional[list[Optional[CsvMappedRow]]] gnomad_data: Optional[list[Optional[GnomADVariant]]] - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] + + +def _scope_query( + query: Select[Any], *, score_set_id: Optional[int], variant_ids: Optional[Sequence[int]] +) -> Select[Any]: + """Narrow *query* to one score set (ordered by variant number) or an explicit variant set.""" + if score_set_id is not None: + return query.where(Variant.score_set_id == score_set_id).order_by(*_VARIANT_NUMBER_ORDER) + return query.where(Variant.id.in_(variant_ids or [])) + + +def _restore_variant_id_order(rows: list[Any], variant_ids: Sequence[int], key) -> list[Any]: + """Postgres does not preserve IN-list order, so restore the caller's ordering by variant id.""" + position = {variant_id: index for index, variant_id in enumerate(variant_ids)} + return sorted(rows, key=lambda row: position.get(key(row), len(position))) + + +def _apply_pagination(query: Select[Any], start: Optional[int], limit: Optional[int]) -> Select[Any]: + """Apply the shared ``start``/``limit`` window (offset/limit) to a variant query. A falsy ``start`` or + ``limit`` leaves that bound off.""" + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + return query + + +def _substrate_query( + *, score_set_id: Optional[int], variant_ids: Optional[Sequence[int]], as_of: Optional[datetime] +) -> Select[Any]: + """The base per-variant query: each variant LEFT-joined to its live mapping record, its authoritative + (measured) allele (digest / ClinGen id / VEP consequence), and that allele's measured projection group + (the mapped HGVS at every level). LEFT joins with ``live_at`` in the ON clause keep unmapped variants + (with null mapped fields); a WHERE would collapse them out. The gnomAD/ClinVar dimensions are resolved by + separate fetches and stitched on by id. + """ + query = ( + select( + Variant, + MappingRecord.id.label("mapping_record_id"), + MappingRecord.assay_level, + MappingRecord.hgvs_assay_level, + Allele.id.label("allele_id"), + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + ProjectionGroup.hgvs_c, + ProjectionGroup.hgvs_g, + ProjectionGroup.hgvs_p, + ) + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .outerjoin( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)), + ) + .outerjoin( + ProjectionGroup, + and_(ProjectionGroup.id == MappingRecordAllele.projection_group, ProjectionGroup.live_at(as_of)), + ) + ) + return _scope_query(query, score_set_id=score_set_id, variant_ids=variant_ids) + + +def _mapped_row(row: Any) -> CsvMappedRow: + """Assemble one variant's :class:`CsvMappedRow` from a substrate-query row: the measured projection + group's HGVS at each level plus the assay-level string, VRS id, VEP consequence, and ClinGen id off the + authoritative allele.""" + return CsvMappedRow( + hgvs_g=row.hgvs_g, + hgvs_c=row.hgvs_c, + hgvs_p=row.hgvs_p, + hgvs_assay_level=row.hgvs_assay_level, + vrs_digest=row.vrs_digest, + vep_functional_consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + ) + + +def _gnomad_by_allele(db: Session, allele_ids: list[int], *, as_of: Optional[datetime]) -> dict[int, GnomADVariant]: + """The live gnomAD frequency record for each of *allele_ids*, keyed by allele id. + + Liveness alone selects the release; there is deliberately no ``db_version`` predicate. The link is + allele-keyed and single-live (``uq_gnomad_allele_links_live``), and ingestion supersedes on that key, + so a release bump retires the prior link rather than accumulating one per version — whatever is live + *is* the release we hold for that allele. Filtering on the currently-served constant on top of that + was wrong in two ways: + + - The linker only visits alleles present in the release it ingests, so an allele absent from the new + release keeps its prior-release link live. Mixed versions are the steady state, not a transient, and + the constant turned every straggler's frequency into a permanent NA. + - Under ``as_of`` the constant is *today's* release, so reconstructing a past instant found the + then-live link and then discarded it — a silent dropout rather than a reconstruction. + + ``db_name`` is still constrained: that is source identity, not a version axis. The release each row + came from is reported in the ``gnomad.gnomad_version`` column, so the output stays self-describing. + """ + if not allele_ids: + return {} + + rows = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + .where(GnomADVariant.db_name == GNOMAD_DB_NAME) + ) + return {allele_id: gnomad_variant for allele_id, gnomad_variant in rows.tuples()} + + +def _clinvar_by_allele( + db: Session, allele_ids: list[int], clinvar_namespaces: dict[str, str], *, as_of: Optional[datetime] +) -> dict[str, dict[int, ClinvarControl]]: + """For each requested ClinVar namespace, the live control at that release for each of *allele_ids* + (``allele id -> control``). ClinVar links are multi-live (one per release), so each namespace's query + is narrowed to its own ``db_version``.""" + by_namespace: dict[str, dict[int, ClinvarControl]] = {} + for ns, db_version in clinvar_namespaces.items(): + per_allele: dict[int, ClinvarControl] = {} + if allele_ids: + rows = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(ClinvarControl.db_name == CLINVAR_DB_NAME, ClinvarControl.db_version == db_version) + ) + per_allele = {allele_id: control for allele_id, control in rows.tuples()} + + by_namespace[ns] = per_allele + + return by_namespace def fetch_variant_csv_data( @@ -36,18 +185,20 @@ def fetch_variant_csv_data( *, score_set: Optional[ScoreSet] = None, variant_ids: Optional[Sequence[int]] = None, - mapped_variant_ids: Optional[Sequence[int]] = None, start: Optional[int] = None, limit: Optional[int] = None, + as_of: Optional[datetime] = None, ) -> CsvFetchResult: """Fetch variant data from the database for CSV generation. Args: score_set: every variant in a score set, ordered by URN suffix. Mutually exclusive with *variant_ids*, which returns an explicit set in the order given. Exactly one is required. - mapped_variant_ids: pins which mapping stands for each variant. Required from a caller that has - already chosen one, since re-resolving on ``current`` alone could pick a different row and - emit a variant twice — nothing in the schema stops two mappings claiming to be current. + as_of: reconstruct the mapping-derived namespaces (reference HGVS, VEP, gnomAD, ClinVar) as they + stood at this instant, over the variant's immutable submitted HGVS/scores/counts. Defaults to + currently-live rows. The deterministic replacement for the old ``current``-flag pinning: a + live mapping record and its authoritative allele link are unique by construction, so there is + no race to guard against by pinning specific row ids. """ if (score_set is None) == (variant_ids is None): raise ValueError("exactly one of score_set or variant_ids must be provided") @@ -60,117 +211,69 @@ def fetch_variant_csv_data( need_gnomad = any(spec.needs_gnomad for spec in specs) need_score_set = any(spec.needs_score_set for spec in specs) - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = select(*select_columns) - - if score_set is not None: - query = query.where(Variant.score_set_id == score_set.id).order_by( - cast(func.split_part(Variant.urn, "#", 2), Integer) - ) - else: - query = query.where(Variant.id.in_(variant_ids or [])) - - if need_score_set: - query = query.options( + score_set_id = score_set.id if score_set is not None else None + score_set_options = ( + [ selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), - ) + ] + if need_score_set + else [] + ) - if need_mappings: - mapping_on_clause = ( - and_(Variant.id == MappedVariant.variant_id, MappedVariant.id.in_(mapped_variant_ids)) - if mapped_variant_ids is not None - else and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)) - ) - query = query.join(MappedVariant, mapping_on_clause, isouter=True) - - # Version predicate belongs in the ON clause: in a WHERE it would drop any variant linked only to - # other-version gnomAD records from the CSV entirely, instead of reporting its frequency as NA. - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant).and_( - GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION + if not need_mappings: + # Fast path: scores/counts (and bare core/score-set identity) are a straight paginated scan, + # untouched by as_of (it only reconstructs the — absent here — mapping-derived namespaces). + query = _apply_pagination( + _scope_query(select(Variant), score_set_id=score_set_id, variant_ids=variant_ids).options( + *score_set_options ), - isouter=True, + start, + limit, ) + result = list(db.scalars(query).all()) + if variant_ids is not None: + result = _restore_variant_id_order(result, variant_ids, key=lambda v: v.id) + return CsvFetchResult(result, None, None, None) - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) + query = _apply_pagination( + _substrate_query(score_set_id=score_set_id, variant_ids=variant_ids, as_of=as_of).options(*score_set_options), + start, + limit, + ) + rows = list(db.execute(query).all()) + if variant_ids is not None: + rows = _restore_variant_id_order(rows, variant_ids, key=lambda row: row.Variant.id) - result = db.execute(query).all() + # Protein slot: the standalone protein subquery over the whole set, stitched back by record id. + # The authoritative allele ids on this page — the join key for the batch gnomAD/ClinVar fetches. + allele_ids = [row.allele_id for row in rows if row.allele_id is not None] + gnomad_by_allele = _gnomad_by_allele(db, allele_ids, as_of=as_of) if need_gnomad else {} + clinvar_by_allele = _clinvar_by_allele(db, allele_ids, clinvar_namespaces, as_of=as_of) - # Postgres does not preserve IN-list order, so restore the caller's ordering. - if variant_ids is not None: - position = {variant_id: index for index, variant_id in enumerate(variant_ids)} - result = sorted(result, key=lambda row: position.get(row[0].id, len(position))) - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data_list is not None: - idx = 2 if need_mappings else 1 - gnomad_data_list.append(row[idx]) - - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - - # One query per namespace, since each names a different release; keyed by MappedVariant id and - # projected back onto row order below. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == CLINVAR_DB_NAME, - aliased_cc.db_version == db_version, - ) - ) - ) - - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - - clinvar_data_map[ns] = mv_to_cc - - clinvar_per_variant = [ - { - ns: mv_to_cc.get(mapping.id) if mapping is not None and mapping.id is not None else None - for ns, mv_to_cc in clinvar_data_map.items() - } - for mapping in mappings - ] + variants: list[Variant] = [] + mappings: list[Optional[CsvMappedRow]] = [] + gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = ( + [] if clinvar_namespaces else None + ) + for row in rows: + variants.append(row.Variant) + allele_id = row.allele_id + mappings.append(_mapped_row(row) if row.mapping_record_id is not None else None) + if gnomad_data is not None: + gnomad_data.append(gnomad_by_allele.get(allele_id) if allele_id is not None else None) + if clinvar_per_variant is not None: + clinvar_per_variant.append( + { + ns: (clinvar_by_allele[ns].get(allele_id) if allele_id is not None else None) + for ns in clinvar_namespaces + } + ) return CsvFetchResult( variants=variants, mappings=mappings, - gnomad_data=gnomad_data_list, + gnomad_data=gnomad_data, clinvar_per_variant=clinvar_per_variant, ) diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py index bbaf33c13..f79f05983 100644 --- a/src/mavedb/lib/csv/score_set.py +++ b/src/mavedb/lib/csv/score_set.py @@ -1,5 +1,6 @@ """The score-set CSV export: every variant in one score set, and the columns it can offer.""" +from datetime import datetime from typing import List, Optional from sqlalchemy import and_, select @@ -38,6 +39,7 @@ def get_score_set_variants_as_csv( start: Optional[int] = None, limit: Optional[int] = None, drop_unused_hgvs_columns_flag: Optional[bool] = None, + as_of: Optional[datetime] = None, viewer: Optional[ScoreCalibrationViewer] = None, ) -> str: """Get the variant data from a score set as a CSV string.""" @@ -53,6 +55,7 @@ def get_score_set_variants_as_csv( score_set=score_set, start=start, limit=limit, + as_of=as_of, ) mappings = fetched.mappings or [None] * len(fetched.variants) @@ -78,12 +81,19 @@ def available_score_set_csv_namespaces( db: Session, score_set: ScoreSet, viewer: Optional[ScoreCalibrationViewer] = None, + *, + as_of: Optional[datetime] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. Its own endpoint rather than a field on the score-set response: it costs several queries and is only needed when a download dialog opens. A namespace absent here is still accepted by the CSV endpoint; it just produces a column of NA. + + ``as_of`` must match the instant the caller will download at. Discovery pinned to "now" against an + ``as_of`` download disagrees in both directions: it offers mapping namespaces that come back entirely + NA, and omits ones the download could have filled. The ClinVar case is the sharpest, since the live + release set decides which ``clinvar.YYYY_MM`` headers exist at all. """ dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} # TODO(#372): non-null id fields @@ -101,12 +111,12 @@ def available_score_set_csv_namespaces( entries.append(static_namespace_entry(CsvNamespace.SCORE_SET)) # always its own provenance - if score_sets_have_current_mappings(db, score_set_ids): + if score_sets_have_current_mappings(db, score_set_ids, as_of=as_of): entries.extend( static_namespace_entry(ns) for ns in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.GNOMAD, CsvNamespace.CLINGEN) ) - entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids))) + entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids, as_of=as_of))) # Every calibration the score set defines is offered, rangeless ones included. calibrations = db.scalars( diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index 99484cdcf..db89b5234 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -8,10 +8,25 @@ from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN from mavedb.lib.validation.constants.general import hgvs_nt_column, hgvs_pro_column, hgvs_splice_column -from mavedb.lib.variants import get_hgvs_from_post_mapped, get_id_from_post_mapped, is_hgvs_g, is_hgvs_p -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant + +@dataclass(frozen=True) +class CsvMappedRow: + """The mapping-derived CSV fields for one variant, resolved from its live mapping record's + authoritative allele (post-mapped HGVS per level, assay-level HGVS, VRS digest, VEP consequence, + ClinGen id). The per-level HGVS strings are the measured projection group's — no raw VRS payload to + fall back on parsing here, unlike the pre-allele-graph ``MappedVariant`` substrate.""" + + hgvs_g: Optional[str] + hgvs_c: Optional[str] + hgvs_p: Optional[str] + hgvs_assay_level: Optional[str] + vrs_digest: Optional[str] + vep_functional_consequence: Optional[str] + clingen_allele_id: Optional[str] + + # One entry per namespace, so adding one is a single edit. This previously took three unrelated changes # — column plan, row builder, fetch-layer eager loading — with nothing to catch a partial addition. @@ -110,55 +125,6 @@ def resolver(self, column_key: str) -> Optional[Callable]: return _optional(lambda data: data.get(column_key)) -def _safe_hgvs_from_post_mapped(mapping: MappedVariant) -> Optional[str]: - """``get_hgvs_from_post_mapped`` with its raises absorbed, since a cell cannot afford to raise. - - ``hgvs_from_vrs_allele`` raises on two payload shapes: a VRS 1.x object, which it refuses - deliberately, and an allele with no ``expressions`` key. Raised from inside a cell resolver, either - would abort the export of every other row in the file. Returning None keeps the refusal — no HGVS is - emitted for those payloads — without letting one variant's shape cost the caller the whole download. - - Intentionally scoped to the CSV export layer, where a single problematic variant should not abort the entire export. - """ - if not mapping.post_mapped: - return None - try: - return get_hgvs_from_post_mapped(mapping.post_mapped) - except (KeyError, ValueError): - return None - - -def _post_mapped_hgvs_g(mapping: Optional[MappedVariant]) -> Optional[str]: - """The genomic HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" - if mapping is None: - return None - if mapping.hgvs_g: - return str(mapping.hgvs_g) - fallback = _safe_hgvs_from_post_mapped(mapping) - return fallback if fallback is not None and is_hgvs_g(fallback) else None - - -def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: - """The protein HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" - if mapping is None: - return None - if mapping.hgvs_p: - return str(mapping.hgvs_p) - fallback = _safe_hgvs_from_post_mapped(mapping) - return fallback if fallback is not None and is_hgvs_p(fallback) else None - - -def _post_mapped_vrs_id(mapping: Optional[MappedVariant]) -> Optional[str]: - """The GA4GH identifier of the post-mapped VRS object, or None if there is no post-mapped object. - - The full ``ga4gh:VA.{digest}`` identifier rather than the bare digest, so a value copied out of an - export can be pasted straight into the VRS search, which validates against the GA4GH CURIE form. - """ - if mapping is None or not mapping.post_mapped: - return None - return get_id_from_post_mapped(mapping.post_mapped) - - def _target_genes(variant: Variant) -> Optional[str]: """The target genes of a variant's score set, joined by ``"; "`` or None if there are none.""" if not variant.score_set: @@ -193,16 +159,14 @@ def _optional(getter: Callable) -> Callable: emit_under=CsvNamespace.SCORES, ), CsvNamespace.COUNTS: CsvNamespaceSpec(source=RowSource.COUNT_DATA, dataset_columns_key="count_columns"), - # TODO(#784): under the allele-centric (RT) substrate these move off MappedVariant onto the Allele. - # Both are reached through `mapping`, so each becomes a one-line hop, not a column-contract change. CsvNamespace.REFERENCE_HGVS: CsvNamespaceSpec( source=RowSource.MAPPING, resolvers={ - "post_mapped_hgvs_g": _post_mapped_hgvs_g, - "post_mapped_hgvs_p": _post_mapped_hgvs_p, - "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), - "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), - "post_mapped_vrs_id": _post_mapped_vrs_id, + "post_mapped_hgvs_g": _optional(attrgetter("hgvs_g")), + "post_mapped_hgvs_p": _optional(attrgetter("hgvs_p")), + "post_mapped_hgvs_c": _optional(attrgetter("hgvs_c")), + "post_mapped_hgvs_at_assay_level": _optional(attrgetter("hgvs_assay_level")), + "post_mapped_vrs_id": _optional(attrgetter("vrs_digest")), }, needs_mappings=True, ), diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py index 61e657e76..932abea74 100644 --- a/src/mavedb/lib/csv/variant.py +++ b/src/mavedb/lib/csv/variant.py @@ -9,6 +9,7 @@ """ import logging +from datetime import datetime from typing import Any, Callable, Optional from sqlalchemy import and_, select @@ -40,7 +41,9 @@ from mavedb.lib.mave.utils import NA_VALUE from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet @@ -88,13 +91,15 @@ def _equivalent_measurements( db: Session, variant_urn: str, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, -) -> Optional[list[tuple[int, int, int]]]: + *, + as_of: Optional[datetime] = None, +) -> Optional[list[tuple[int, int]]]: """Resolve a variant URN to the measurements the CSV should report. - Returns ``[(variant_id, mapped_variant_id, score_set_id), ...]``, requested variant first, then every - other current measurement of the same ClinGen allele ordered by score set and variant URN so repeated - downloads are byte-identical. At most one entry per variant, which is what lets the fetch layer - restore this order from variant ids alone. + Returns ``[(variant_id, score_set_id), ...]``, requested variant first, then every other measurement + live as of *as_of* against the same ClinGen allele, ordered by score set and variant URN so repeated + downloads are byte-identical. A live mapping record and its authoritative allele link are unique by + construction, so at most one entry exists per variant without needing to pin or deduplicate. Args: may_read_score_set: when given, drops measurements from score sets the caller may not read. @@ -102,88 +107,93 @@ def _equivalent_measurements( requested variant is not repeated here. Returns: - None when the variant has no current mapping, since there is then no allele to expand by. + None when the variant has no live mapping as of *as_of*, since there is then no allele to expand by. """ - # Nothing in the schema enforces one current mapping per variant, and every column below follows from - # this pick, so order explicitly: an unordered LIMIT 1 would let one URN download differently twice. - requested = db.scalars( - select(MappedVariant) - .join(MappedVariant.variant) - .where(and_(Variant.urn == variant_urn, MappedVariant.current.is_(True))) - .order_by(MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + requested = db.execute( + select(Variant.id, Variant.score_set_id, Allele.clingen_allele_id) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(Variant.urn == variant_urn) .limit(1) ).one_or_none() if requested is None: return None + requested_variant_id, requested_score_set_id, clingen_allele_id = requested + # TODO(#372): non-null id fields - if not requested.clingen_allele_id: - return [(requested.variant_id, requested.id, requested.variant.score_set_id)] # type: ignore + if not clingen_allele_id: + return [(requested_variant_id, requested_score_set_id)] # type: ignore - # TODO(#784): under the allele-centric (RT) substrate, replace this string match with the - # projection-aware resolver — measurements attach to an Allele and c/g pairs relate through - # `projection_group`. Only this query changes; callers consume (variant_id, mapped_variant_id) pairs. equivalents = db.execute( - select(MappedVariant.variant_id, MappedVariant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) - .join(MappedVariant.variant) + select(Variant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) .join(Variant.score_set) .where( and_( - MappedVariant.clingen_allele_id == requested.clingen_allele_id, - MappedVariant.current.is_(True), - # By variant, not by mapping: the anchor already represents this variant, and a second - # current mapping on it is the same measurement again, not an equivalent one. - MappedVariant.variant_id != requested.variant_id, + Allele.clingen_allele_id == clingen_allele_id, + # By variant, not by mapping: the anchor already represents this variant, and its own + # live mapping is the same measurement again, not an equivalent one. + Variant.id != requested_variant_id, ) ) - .order_by(MappedVariant.variant_id, MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) ).all() - # One row per variant, picked as the anchor was. A duplicate would defeat the downstream row-order - # restoration, which keys on variant id. TODO(#784): moot once a measurement points at one Allele. - deduplicated: dict[int, Any] = {} - for row in equivalents: - deduplicated.setdefault(row[0], row) - equivalents = list(deduplicated.values()) - if may_read_score_set is not None and equivalents: - candidate_score_set_ids = {row[2] for row in equivalents} + candidate_score_set_ids = {row[1] for row in equivalents} readable_score_set_ids = { score_set.id for score_set in db.scalars(select(ScoreSet).where(ScoreSet.id.in_(candidate_score_set_ids))).all() if may_read_score_set(score_set) } - equivalents = [row for row in equivalents if row[2] in readable_score_set_ids] + equivalents = [row for row in equivalents if row[1] in readable_score_set_ids] equivalents = sorted( equivalents, - key=lambda row: (score_set_urn_sort_key(row[3]), variant_urn_sort_key(row[4])), + key=lambda row: (score_set_urn_sort_key(row[2]), variant_urn_sort_key(row[3])), ) - return [(requested.variant_id, requested.id, requested.variant.score_set_id)] + [ - (row[0], row[1], row[2]) for row in equivalents - ] + return [(requested_variant_id, requested_score_set_id)] + [(row[0], row[1]) for row in equivalents] -def _unmapped_variant_namespaces(db: Session, score_set_id: int) -> list[str]: - """The namespaces to offer for a variant that exists but has no current mapping. +def _unmapped_variant_namespaces(db: Session, score_set_id: int, *, as_of: Optional[datetime] = None) -> list[str]: + """The namespaces to offer for a variant that exists but has no mapping live as of *as_of*. Always score, score set, and relationship; plus the mapping-derived groups when the score set has been mapped at all, where NA is the honest value for a variant the mapper has not reached. """ - if score_sets_have_current_mappings(db, [score_set_id]): + if score_sets_have_current_mappings(db, [score_set_id], as_of=as_of): return list(BASE_VARIANT_CSV_NAMESPACES) return list(ALWAYS_AVAILABLE_NAMESPACES) -def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional[str]: +def _latest_clinvar_namespace( + db: Session, score_set_ids: list[int], *, as_of: Optional[datetime] = None +) -> Optional[str]: """The ClinVar namespace for the most recent release covering these measurements' score sets. One release for the whole file rather than each variant's own latest: that keeps the release in the column header where it is citable, and avoids mixing calls from different releases in one column. """ - namespaces = clinvar_release_namespaces(db, score_set_ids) + namespaces = clinvar_release_namespaces(db, score_set_ids, as_of=as_of) if not namespaces: return None @@ -236,6 +246,8 @@ def available_variant_csv_namespaces( variant_urn: str, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, viewer: Optional[ScoreCalibrationViewer] = None, + *, + as_of: Optional[datetime] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. @@ -245,7 +257,7 @@ def available_variant_csv_namespaces( Raises: ValueError: if no variant with *variant_urn* exists. """ - measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set, as_of=as_of) if measurements is None: variant = db.scalars(select(Variant).where(Variant.urn == variant_urn).limit(1)).first() @@ -255,16 +267,19 @@ def available_variant_csv_namespaces( # No mapping on this variant, but its score set may still be mapped, in which case the # mapping-derived columns are owed with NA rather than omitted. # TODO(#372): non-null id fields - return [static_namespace_entry(ns) for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id))] # type: ignore + return [ + static_namespace_entry(ns) + for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id), as_of=as_of) # type: ignore + ] base_entries = [static_namespace_entry(ns) for ns in BASE_VARIANT_CSV_NAMESPACES] - score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + score_set_ids = list({score_set_id for _, score_set_id in measurements}) return ( base_entries + calibration_namespace_entries(_annotatable_calibration_namespaces(db, score_set_ids, viewer).values()) - + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) + + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids, as_of=as_of)) ) @@ -275,33 +290,36 @@ def get_variant_csv( may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, viewer: Optional[ScoreCalibrationViewer] = None, na_rep: str = NA_VALUE, + *, + as_of: Optional[datetime] = None, ) -> str: """Build the clinical CSV for a variant and its equivalent measurements. - One row per measurement: the requested variant first, then every other current measurement of the - same ClinGen allele across score sets. + One row per measurement: the requested variant first, then every other measurement live as of + *as_of* against the same ClinGen allele, across score sets. Args: namespaces: columns to include, in the same vocabulary as the score-set CSV. When omitted, defaults to the fixed namespaces plus every eligible calibration and the latest ClinVar release covering these measurements. + as_of: reconstruct every measurement (and its mapping-derived columns) as it stood at this + instant. Defaults to currently-live rows. Raises: ValueError: if no variant with *variant_urn* exists. """ - measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set, as_of=as_of) if measurements is None: - return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep) + return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep, as_of=as_of) - variant_ids = [variant_id for variant_id, _, _ in measurements] - mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] - score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + variant_ids = [variant_id for variant_id, _ in measurements] + score_set_ids = list({score_set_id for _, score_set_id in measurements}) calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, viewer) if namespaces is None: - clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) + clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids, as_of=as_of) # Research-use-only calibrations are offerable but never defaulted to. This export is framed clinically. default_calibrations = sorted( namespace for namespace, calibration in calibrations_by_ns.items() if not calibration.research_use_only @@ -321,7 +339,7 @@ def get_variant_csv( columns, plan.clinvar_namespaces, variant_ids=variant_ids, - mapped_variant_ids=mapped_variant_ids, + as_of=as_of, ) mappings = fetched.mappings or [None] * len(fetched.variants) @@ -346,8 +364,10 @@ def _unmapped_variant_csv( variant_urn: str, namespaces: Optional[list[str]] = None, na_rep: str = NA_VALUE, + *, + as_of: Optional[datetime] = None, ) -> str: - """Build the single-row CSV for a variant that exists but has no current mapping. + """Build the single-row CSV for a variant that exists but has no mapping live as of *as_of*. Without a mapping there are no coordinates, no external annotations, no allele to find equivalents by, and nothing for the annotation layer to flatten. Identity, score, and provenance still resolve, so the @@ -368,7 +388,9 @@ def _unmapped_variant_csv( dataset_columns={}, namespaces=( # TODO(#372): non-null id fields - list(namespaces) if namespaces is not None else _unmapped_variant_namespaces(db, int(variant.score_set_id)) # type: ignore + list(namespaces) + if namespaces is not None + else _unmapped_variant_namespaces(db, int(variant.score_set_id), as_of=as_of) # type: ignore ), ) columns = plan.namespaced_columns diff --git a/src/mavedb/lib/deprecation.py b/src/mavedb/lib/deprecation.py new file mode 100644 index 000000000..b99eed244 --- /dev/null +++ b/src/mavedb/lib/deprecation.py @@ -0,0 +1,81 @@ +"""Build RFC 8594 deprecation headers for retired routes, parameters, and artifacts. + +RFC 8594 defines ``Deprecation`` (the resource is deprecated) and ``Sunset`` (when it will stop working). +:func:`deprecation_headers` emits ``Deprecation: true`` and, where a replacement exists, a ``Link`` with +``rel="successor-version"``; a ``Warning: 299`` carries human-readable guidance for clients that do not +parse the structured headers. ``Sunset`` is included only when a removal date is supplied. +""" + +import logging +from typing import Optional + +from mavedb.lib.logging.context import logging_context, save_to_logging_context + +logger = logging.getLogger(__name__) + +# The removal date for the MappedVariant-retirement surfaces (the /mapped-variants redirects and the +# score-set tombstone), as an IMF-fixdate string (e.g. "Wed, 01 Jul 2026 00:00:00 GMT"), or None to emit no +# Sunset header. Unset until the team ratifies a date. Scoped to this one campaign — a different deprecation +# should not inherit this date, so pass it explicitly at each MappedVariant-retirement call site rather than +# relying on a builder-wide default. +MAPPED_VARIANT_SUNSET: Optional[str] = None + +# Where an integrator should read about the migration. Points at the dump/API changelog channel. +DEPRECATION_DOCS_URL = "https://mavedb.org/docs/mavedb/changelog.html" + + +def deprecation_headers( + *, + successor: Optional[str] = None, + warning: Optional[str] = None, + sunset: Optional[str] = None, + docs_url: Optional[str] = DEPRECATION_DOCS_URL, +) -> dict[str, str]: + """Build the RFC 8594 header set for a deprecated resource. + + Args: + successor: absolute path or URL of the 1:1 replacement resource, emitted as a + ``Link: ; rel="successor-version"``. Omit when no wire-compatible successor exists + (a cardinality change), and rely on ``warning`` to point the caller at the nearest resource. + warning: human-readable guidance, emitted as ``Warning: 299 - ""``. + sunset: IMF-fixdate removal date, emitted as ``Sunset``. No default — each deprecation campaign has + its own removal timeline, so callers pass their own date constant explicitly. + docs_url: a documentation link, emitted as ``Link: ; rel="deprecation"``. Pass None to omit. + """ + headers: dict[str, str] = {"Deprecation": "true"} + + links = [] + if successor: + links.append(f'<{successor}>; rel="successor-version"') + if docs_url: + links.append(f'<{docs_url}>; rel="deprecation"') + if links: + headers["Link"] = ", ".join(links) + + if sunset: + headers["Sunset"] = sunset + if warning: + headers["Warning"] = f'299 - "{warning}"' + + return headers + + +def record_deprecated_usage(surface: str, *, successor: Optional[str] = None) -> None: + """Record that a request touched a deprecated surface, under one queryable log marker. + + Every deprecated surface — a retired route, a deprecated query parameter, a future one — should show up + the same way, so "who is still on a deprecated surface?" is a single query rather than a per-surface hunt. + + Stamps ``deprecation_marker: True`` into the logging context (the aggregation key), accumulates ``surface`` + into ``deprecated_surfaces`` so several deprecations on one request all show, and logs a warning naming the + surface and its successor. ``surface`` is the thing used (a route path or a parameter name); ``successor`` + is its replacement, if one exists. + """ + surfaces = sorted({*logging_context().get("deprecated_surfaces", []), surface}) + save_to_logging_context({"deprecated_surfaces": surfaces, "deprecation_marker": True}) + logger.warning( + "Request used the deprecated surface %r; it will be removed in a future release.%s", + surface, + f" Use {successor} instead." if successor else "", + extra={"deprecated_surface": surface, "deprecation_successor": successor}, + ) diff --git a/src/mavedb/lib/exceptions.py b/src/mavedb/lib/exceptions.py index 8b8633f1f..e0c079a67 100644 --- a/src/mavedb/lib/exceptions.py +++ b/src/mavedb/lib/exceptions.py @@ -222,6 +222,12 @@ class HGNCServiceError(Exception): pass +class MondoServiceError(Exception): + """Raised when the MONDO/OLS disease ontology service cannot provide search results.""" + + pass + + class LDHSubmissionFailureError(Exception): """Raised when submission to ClinGen Linked Data Hub (LDH) fails for all submissions.""" diff --git a/src/mavedb/lib/experiments.py b/src/mavedb/lib/experiments.py index a200e93bb..22b2c2d1c 100644 --- a/src/mavedb/lib/experiments.py +++ b/src/mavedb/lib/experiments.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import Session from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.collections import readable_official_collections from mavedb.lib.permissions import Action from mavedb.lib.score_sets import find_superseded_score_set_tail from mavedb.lib.types.authentication import UserData @@ -137,6 +138,10 @@ def enrich_experiment_with_num_score_sets( """ Validate and update the number of score set in experiment. The superseded score set is excluded. Data structure: experiment{score_set_urns, num_score_sets} + + Also narrows the experiment's official collections to what the caller may read. Every route that returns + an ``Experiment`` view model must build it here; serializing the ORM experiment directly lists every + score set URN and every collection, private ones included. """ filter_superseded_score_set_tails = [ find_superseded_score_set_tail(score_set, Action.READ, user_data) for score_set in item_update.score_sets @@ -153,6 +158,7 @@ def enrich_experiment_with_num_score_sets( update={ "num_score_sets": len(filtered_score_set_urns), "score_set_urns": filtered_score_set_urns, + "official_collections": readable_official_collections(user_data, item_update.official_collections), } ) return updated_experiment diff --git a/src/mavedb/lib/external_publications.py b/src/mavedb/lib/external_publications.py index a48ff7185..326102d77 100644 --- a/src/mavedb/lib/external_publications.py +++ b/src/mavedb/lib/external_publications.py @@ -564,4 +564,5 @@ def _fetch(self, url: str, return_format: str = "json") -> Any: return [] response.raise_for_status() + return json.loads(response.text) if return_format == "json" else response.text diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 9bfa0fec9..c536231eb 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -1,24 +1,119 @@ import logging import os import re -from typing import Any, Sequence, Union - -from sqlalchemy import Connection, Row, select, text +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Any, Optional, Sequence, Union +from urllib.parse import quote + +from sqlalchemy import Connection, Row, func, select, text from sqlalchemy.orm import Session -from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.utils import batched -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant -GNOMAD_DB_NAME = "gnomAD" -GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") # e.g., "v4.1" logger = logging.getLogger(__name__) +GNOMAD_DB_NAME = "gnomAD" +GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") +GNOMAD_BROWSER_URL = "https://gnomad.broadinstitute.org" +_CAID_LEADING_ZERO_RE = r"^(CA)0+([0-9])" +""" +Strip leading zeros from a CAID's numeric portion, keeping at least one digit. +Kept byte-for-byte in sync with the SQL form used to normalize Allele.clingen_allele_id +in link_gnomad_variants_to_alleles. +""" +_GNOMAD_BROWSER_DATASETS = {"2": "gnomad_r2_1", "3": "gnomad_r3", "4": "gnomad_r4"} +"""Mapping of gnomAD major release numbers to browser dataset names.""" + + +class GnomadLinkVerdict(str, Enum): + """Per-allele outcome of a gnomAD linking run, returned for every allele the linker touched. + + The single source of truth for what happened to an allele's link this run — the caller derives + annotation status directly from this, never by re-querying link state (which would be a second, + drift-prone source of truth). + """ + + CREATED = "created" # link created or superseded this run (a new/changed live link) + UNCHANGED = "unchanged" # a live link already pointed at the resolved variant; left untouched + + +@dataclass(frozen=True) +class GnomadVariantLink: + """One (score-set variant, annotated allele) pair a gnomAD frequency record reaches. + + Mirrors :class:`mavedb.lib.clinical_controls.ControlVariantLink`. ``allele_digest`` is the VRS digest + of the allele gnomAD's frequency was linked to. + """ + + variant_urn: str + allele_digest: str + + +def get_gnomad_variants_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_version: Optional[str] = None, +) -> list[tuple[GnomADVariant, list[GnomadVariantLink]]]: + """The live gnomAD frequency records for a score set, each paired with the score-set variants that link + to it. + + Walks ``GnomadAlleleLink → Allele → MappingRecordAllele → MappingRecord → Variant`` with every + ``ValidTime`` hop constrained to the same instant (``as_of``, defaulting to currently-live). Optionally + narrowed to one gnomAD release (``db_version``). Records come back in first-seen order; each + record's links preserve query order. + """ + stmt = ( + select( + GnomADVariant, + Variant.urn.label("variant_urn"), + Allele.vrs_digest.label("allele_digest"), + ) + .join(GnomadAlleleLink, GnomadAlleleLink.gnomad_variant_id == GnomADVariant.id) + .join(Allele, Allele.id == GnomadAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(GnomadAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + if db_version is not None: + stmt = stmt.where(GnomADVariant.db_version == db_version) + + variants: dict[int, GnomADVariant] = {} + links_by_variant: dict[int, list[GnomadVariantLink]] = {} + + for gnomad_variant, variant_urn, allele_digest in db.execute(stmt).tuples(): + # TODO#372: nullable URN + if variant_urn is None: + continue + + if gnomad_variant.id not in variants: + variants[gnomad_variant.id] = gnomad_variant + links_by_variant[gnomad_variant.id] = [] + + links_by_variant[gnomad_variant.id].append( + GnomadVariantLink(variant_urn=variant_urn, allele_digest=allele_digest) + ) + + return [(gnomad_variant, links_by_variant[gnomad_variant.id]) for gnomad_variant in variants.values()] + + def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str]) -> str: """ Generate a gnomAD variant identifier based on contig, position, and alleles. @@ -33,6 +128,18 @@ def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str] return f"{contig}-{position}-{'-'.join(alleles)}" +def gnomad_variant_url(db_identifier: str, db_version: str) -> str: + """Browser link for a gnomAD variant, pinned to the dataset of ``db_version`` (e.g. ``v4.1``). + + An unrecognized ``db_version`` gets no ``dataset`` parameter rather than a guessed one. Mirrors + ``gnomadVariantUrl`` in the UI's ``src/lib/gnomad.ts``. + """ + url = f"{GNOMAD_BROWSER_URL}/variant/{quote(db_identifier, safe='')}" + major = re.match(r"v?(\d+)", db_version) + dataset = _GNOMAD_BROWSER_DATASETS.get(major.group(1)) if major else None + return f"{url}?dataset={dataset}" if dataset else url + + def gnomad_table_name() -> str: """ Generate the gnomAD table name based on the data version. @@ -46,6 +153,18 @@ def gnomad_table_name() -> str: return table_name +def normalize_caid(caid: str) -> str: + """Normalize a ClinGen CAID by stripping leading zeros from its numeric portion. + + The gnomAD Hail/Athena dump drops leading zeros from CAIDs — MaveDB's ``CA025094`` is recorded as + ``CA25094`` — so an exact-string join silently misses every zero-padded CAID (issue #722). Both + sides of the join are normalized to the unpadded form to repair the match. ``CA025094`` and + ``CA25094`` denote the same ClinGen allele, so this can never collide distinct alleles. A value + that is not a recognizable CAID (no ``CA`` prefix + digits) is returned unchanged. + """ + return re.sub(_CAID_LEADING_ZERO_RE, r"\1\2", caid) + + def allele_list_from_list_like_string(alleles_string: str) -> list[str]: """ Convert a list-like string representation of alleles into a Python list. @@ -94,8 +213,9 @@ def gnomad_variant_data_for_caids( Raises: sqlalchemy.exc.SQLAlchemyError: If there is an error executing the query. """ + # Normalize to the unpadded form the dump stores so the IN-list matches zero-padded CAIDs (see issue #722). chunked_caids = batched(caids, 16250) - caid_strs = [",".join(f"'{caid}'" for caid in chunk) for chunk in chunked_caids] + caid_strs = [",".join(f"'{normalize_caid(caid)}'" for caid in chunk) for chunk in chunked_caids] save_to_logging_context({"num_caids": len(caids), "num_chunks": len(caid_strs)}) result_rows: list[Row[Any]] = [] @@ -132,31 +252,56 @@ def gnomad_variant_data_for_caids( return result_rows -def link_gnomad_variants_to_mapped_variants( - db: Session, gnomad_variant_data: Sequence[Row[Any]], only_current: bool = True -) -> int: - """ - Links gnomAD variants to mapped variants in the database based on CAIDs. Note that this function does - not commit this data to the database; it only prepares the relationships. - - Args: - caids (list[str]): A list of CAIDs to link with gnomAD variants. +def link_gnomad_variants_to_alleles( + db: Session, gnomad_variant_data: Sequence[Row[Any]] +) -> dict[int, GnomadLinkVerdict]: + """Link gnomAD variants to deduplicated alleles by CAID, superseding only on change. + + Every ``Allele`` carrying the row's ``clingen_allele_id`` (populated by CAR) is linked through a + valid-time :class:`GnomadAlleleLink`, so one gnomAD variant fans out to every allele sharing the + CAID (cross-score-set dedup included). Each allele holds at most one live link, superseded **only + on change**: a live link already pointing to the resolved variant is left untouched (an unchanged + re-run writes no spurious valid-time boundary); a new/different/older-version target retires it and + inserts a successor, keyed on ``allele_id`` so a version bump replaces rather than accumulates. The + guard is load-bearing despite the job's upstream skip — shared CAIDs and ``force`` runs still reach + it. A current-version link to a *different* identifier (a CAID re-resolved within one release) is + logged and superseded newest-wins, not raised. + + Does not commit. Returns a verdict per allele *touched* this run (matched a CAID-bearing row): + :attr:`GnomadLinkVerdict.CREATED` for a created/superseded link, :attr:`~GnomadLinkVerdict.UNCHANGED` + for a live link left in place. Alleles absent from the map were matched by no row — the caller reads + those as "gnomAD had no record". This is the single source of truth for per-allele status; callers + must not re-derive it by re-querying link state. """ save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) - save_to_logging_context({"only_current": only_current}) - logger.debug(msg="Linking gnomAD variants to mapped variants", extra=logging_context()) - - linked_gnomad_variants = 0 - annotation_manager = AnnotationStatusManager(db) + logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) + + # Resolve all rows' alleles in one query, then group in Python. The regexp_replace predicate + # (normalizing the zero-padded stored CAID to the dump's stripped form, #722) is non-sargable, so + # a per-row query would seq-scan `alleles` once per variant; one IN scan replaces those N scans. + target_caids = {normalize_caid(row.caid) for row in gnomad_variant_data} + alleles_by_caid: dict[str, list[Allele]] = defaultdict(list) + if target_caids: + for allele in db.scalars( + select(Allele).where( + func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2").in_(target_caids) + ) + ).all(): + # clingen_allele_id is nullable on the model, but the WHERE above only matches rows where + # regexp_replace(clingen_allele_id, ...) is non-null (SQL NULL fails .in_()), so every row + # returned here has one. + assert allele.clingen_allele_id is not None + alleles_by_caid[normalize_caid(allele.clingen_allele_id)].append(allele) + + verdicts: dict[int, GnomadLinkVerdict] = {} for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() ) - mapped_variants_with_caids_query = select(MappedVariant).where(MappedVariant.clingen_allele_id == row.caid) - if only_current: - mapped_variants_with_caids_query = mapped_variants_with_caids_query.where(MappedVariant.current.is_(True)) - mapped_variants_with_caids = db.scalars(mapped_variants_with_caids_query).all() + alleles_with_caid = alleles_by_caid.get(normalize_caid(row.caid), []) + if not alleles_with_caid: + continue gnomad_identifier_for_variant = gnomad_identifier( row.__getattribute__("locus.contig"), @@ -172,80 +317,88 @@ def link_gnomad_variants_to_mapped_variants( if faf95_max is not None: faf95_max = float(faf95_max) - for mapped_variant in mapped_variants_with_caids: - # Remove any existing gnomAD variants for this mapped variant that match the current gnomAD data version to avoid data duplication. - # There should only be one gnomAD variant per mapped variant per gnomAD data version, since each gnomAD variant can only match to one - # CAID. - for linked_gnomad_variant in mapped_variant.gnomad_variants: - if linked_gnomad_variant.db_version == GNOMAD_DATA_VERSION: - logger.debug( - msg=f"Removing existing gnomAD variant {linked_gnomad_variant.db_identifier} from mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", - extra=logging_context(), - ) - mapped_variant.gnomad_variants.remove(linked_gnomad_variant) - - existing_gnomad_variant = db.scalar( - select(GnomADVariant).where( - GnomADVariant.db_name == "gnomAD", - GnomADVariant.db_identifier == gnomad_identifier_for_variant, - GnomADVariant.db_version == GNOMAD_DATA_VERSION, - ) + # One gnomAD variant per (identifier, version): get-or-create so repeated CAIDs and re-runs + # reuse the same row. Flush so a freshly created variant has an id for the link below. + gnomad_variant = db.scalar( + select(GnomADVariant).where( + GnomADVariant.db_name == GNOMAD_DB_NAME, + GnomADVariant.db_identifier == gnomad_identifier_for_variant, + GnomADVariant.db_version == GNOMAD_DATA_VERSION, + ) + ) + if gnomad_variant is None: + logger.debug( + msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), + ) + gnomad_variant = GnomADVariant( + db_name=GNOMAD_DB_NAME, + db_identifier=gnomad_identifier_for_variant, + db_version=GNOMAD_DATA_VERSION, + allele_count=allele_count, + allele_number=allele_number, + allele_frequency=allele_frequency, # type: ignore + faf95_max_ancestry=faf95_max_ancestry, + faf95_max=faf95_max, # type: ignore + ) + db.add(gnomad_variant) + db.flush() + else: + logger.debug( + msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), ) - if existing_gnomad_variant is None: - logger.debug( - msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", - extra=logging_context(), + for allele in alleles_with_caid: + live_link = db.scalar( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, ) - gnomad_variant = GnomADVariant( - db_name=GNOMAD_DB_NAME, - db_identifier=gnomad_identifier_for_variant, - db_version=GNOMAD_DATA_VERSION, - allele_count=allele_count, - allele_number=allele_number, - allele_frequency=allele_frequency, # type: ignore - faf95_max_ancestry=faf95_max_ancestry, - faf95_max=faf95_max, # type: ignore - ) - else: - logger.debug( - msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + ) + + # No change: live link already points here — leave it untouched (no spurious boundary). + if live_link is not None and live_link.gnomad_variant_id == gnomad_variant.id: + verdicts.setdefault(allele.id, GnomadLinkVerdict.UNCHANGED) + continue + + if ( + live_link is not None + and live_link.gnomad_variant.db_version == GNOMAD_DATA_VERSION + and live_link.gnomad_variant.db_identifier != gnomad_identifier_for_variant + ): + logger.warning( + msg=( + f"CAID {allele.clingen_allele_id} for allele {allele.id} resolved to " + f"{gnomad_identifier_for_variant} at version {GNOMAD_DATA_VERSION}, but a live link " + f"already points to {live_link.gnomad_variant.db_identifier} at the same version. " + "Superseding (newest wins); investigate the gnomAD source for a re-resolved CAID." + ), extra=logging_context(), ) - gnomad_variant = existing_gnomad_variant - if gnomad_variant not in mapped_variant.gnomad_variants: - mapped_variant.gnomad_variants.append(gnomad_variant) - linked_gnomad_variants += 1 - - db.add(gnomad_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "gnomad_db_identifier": gnomad_variant.db_identifier, - } - }, - current=True, + # Change: retire any live link for the allele, insert the successor (allele-keyed, so a + # version bump replaces rather than accumulates). + GnomadAlleleLink.supersede_live_where( + db, + [GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)], + GnomadAlleleLink.allele_id == allele.id, ) + verdicts[allele.id] = GnomadLinkVerdict.CREATED # created always wins over a same-run unchanged logger.debug( - msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", + msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to allele {allele.id} ({allele.clingen_allele_id})", extra=logging_context(), ) logger.info( - f"Linked {len(mapped_variants_with_caids)} mapped variants with CAID {row.caid} to gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" + f"Processed {len(alleles_with_caid)} alleles with CAID {row.caid} for gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" ) - annotation_manager.flush() - - save_to_logging_context({"linked_gnomad_variants": linked_gnomad_variants}) + changed_allele_count = sum(1 for v in verdicts.values() if v is GnomadLinkVerdict.CREATED) + save_to_logging_context({"changed_allele_count": changed_allele_count}) logger.info( - msg=f"Linked a total of {linked_gnomad_variants} gnomAD variants to mapped variants.", + msg=f"Created or superseded {changed_allele_count} allele links this run.", extra=logging_context(), ) - return linked_gnomad_variants + return verdicts diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py new file mode 100644 index 000000000..b7f2428f1 --- /dev/null +++ b/src/mavedb/lib/hgvs.py @@ -0,0 +1,194 @@ +import re +import sys +from dataclasses import dataclass +from typing import Optional + +# Coordinate prefix of an HGVS variant description: a single type letter plus a dot +# (g. c. n. m. r. p. ...), capturing the prefix and the remaining description separately. +_HGVS_COORD_PREFIX = re.compile(r"^([a-z]\.)(.+)$") + +_FIRST_INT = re.compile(r"\d+") + + +def _cis_phased_sort_key(description: str) -> tuple[int, str]: + """Order key for a cis-phased component description by its first integer position. + + The first run of digits is the coordinate for both genomic (``123A>G``) and protein + (``Arg123Gly``) forms. Descriptions with no digit sort last, and the raw string breaks ties so + the order is total and stable. + """ + match = _FIRST_INT.search(description) + return (int(match.group()) if match else sys.maxsize, description) + + +def extract_accession(hgvs_string: str) -> str: + """Extract the reference accession from an HGVS string, or return empty string if it cannot + be parsed. + + This function makes no assumptions about the structure of the accession (e.g. whether it starts with "NM_" + or "NC_") and is robust to extra whitespace. It simply returns the substring before the first colon, which + is the standard HGVS separator between the accession and the variant description. Callers must validate + the accession format separately if needed. + """ + token = (hgvs_string or "").strip() + if ":" not in token: + return "" + + return token.split(":", 1)[0].strip() + + +_HGVS_P_PREDICTION = re.compile(r":p\.\((.+)\)\s*$") + + +def strip_protein_prediction_parens(hgvs_p: str) -> str: + """Unwrap the prediction parentheses from a protein HGVS: ``p.(Ala222Val)`` -> ``p.Ala222Val``. + + Forward translation (``c_to_p``) emits a *predicted* consequence in parentheses. ga4gh + translates either form fine, but the parens are noise we don't want in the stored HGVS -- + they denote inference, not a different variant -- so we normalize to the bare form for + consistency. A string with no prediction parens is returned unchanged. + """ + return _HGVS_P_PREDICTION.sub(r":p.\1", hgvs_p) + + +_NT_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?[cgn]\.(\d+)([ACGTacgt])>([ACGTacgt])$") +""" +Nucleotide: only a purely numeric position yields a placeable block — UTR/intron coding positions +(`c.*123`, `c.-12`, `c.12+3`) are not single integers and so are not parsed into a block (the +heatmap cannot place them either). ref/alt are single nucleotides. +""" +_PRO_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?p\.\(?([A-Za-z]{3})(\d+)([A-Za-z]{3}|=|\*|-)\)?$") +""" +Protein: three-letter ref, integer position, and a three-letter alt or one of `=` (synonymous), +`*` (Ter/stop), `-` (deletion) — matching MaveHGVS-pro and the heatmap's amino-acid rows. +""" + + +@dataclass(frozen=True) +class SequenceBlock: + """A parsed single-locus substitution: 1-based ``position`` plus ``ref`` and ``alt`` residues. + + For coding DNA ``ref``/``alt`` are single nucleotides; for protein they are three-letter amino-acid + codes or one of ``=`` (synonymous), ``*`` (Ter/stop), ``-`` (deletion). + """ + + position: int + ref: str + alt: str + + +def parse_simple_nucleotide_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus coding/genomic substitution (``c.1216G>A``) into a :class:`SequenceBlock`. + + Returns ``None`` for anything that is not a numeric-position single-nucleotide substitution + (multivariant, indel, UTR/intron position, empty), so non-placeable variants simply yield no block. + """ + if not hgvs: + return None + match = _NT_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(1)), ref=match.group(2), alt=match.group(3)) + + +def parse_simple_protein_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus protein substitution (``p.Ala406Thr``, ``p.(Ala406Thr)``) into a + :class:`SequenceBlock`. + + The alt is the raw MaveHGVS token: a three-letter code, ``=`` (synonymous), ``*`` (Ter), or ``-`` + (deletion). Returns ``None`` for anything else (multivariant, frameshift, empty). + """ + if not hgvs: + return None + match = _PRO_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(2)), ref=match.group(1), alt=match.group(3)) + + +def parse_simple_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus nucleotide *or* protein substitution, auto-detecting by coordinate prefix. + + The nucleotide (``c.``/``g.``/``n.``) and protein (``p.``) forms are disjoint, so trying both is + unambiguous — useful when the level is not known up front (e.g. the assay-level HGVS, which is + coding for a nucleotide assay and protein for a protein assay). Returns ``None`` for anything that + is not a placeable simple substitution. + """ + return parse_simple_nucleotide_substitution(hgvs) or parse_simple_protein_substitution(hgvs) + + +def is_cis_phased_hgvs(hgvs_string: str) -> bool: + """True for an accession-qualified allele-list expression (``NC_…:g.[123A>G;125T>C]``). + + Keys on the allele-list opener ``.[`` — the bracket immediately after the coordinate prefix — + so tandem-repeat brackets (``c.101_102[4]``, bracket after a position) are not misread as an + allele list. Reverse translation's forward parser rejects the ``[`` opening the list, so callers + skip these inputs rather than send a guaranteed failure to the engine. + """ + _, separator, remainder = hgvs_string.partition(":") + return bool(separator) and ".[" in remainder + + +def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: + """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. + + The reverse-translate-variants tool emits the non-adjacent component substitutions of a + single codon change as one bracketed genomic expression (``NC_000001.11:g.[123A>G;125T>C]``) + whenever they are too far apart on the genome to collapse into a single delins. ga4gh's + AlleleTranslator only ever produces a single Allele, so each component must be translated + independently and recombined into a VRS CisPhasedBlock. + + Each returned component carries the original accession and coordinate prefix + (``NC_000001.11:g.123A>G``). A non-bracketed expression is returned unchanged as a + single-element list, so callers can treat both cases uniformly. + + Unlike a bare mavehgvs split, the accession is preserved: the components feed straight + into VRS translation, which requires a reference accession to resolve positions. + """ + accession, separator, remainder = hgvs_string.partition(":") + # Only an accession-qualified, bracketed expression is a cis-phased multivariant we split here; + # anything else (bare, unbracketed, or accession-less) is returned unchanged so the caller can + # treat both cases uniformly without a ValueError on the missing ":" / "[". + if not separator or "[" not in remainder: + return [hgvs_string] + + prefix = remainder[: remainder.index("[")] # e.g. "g." / "c." + inner = remainder[remainder.index("[") + 1 : remainder.rindex("]")] + return [f"{accession}:{prefix}{component}" for component in inner.split(";") if component] + + +def join_cis_phased_hgvs(components: list[str]) -> Optional[str]: + """Join cis-phased component HGVS strings into one bracketed expression. + + Inverse of :func:`split_cis_phased_hgvs`: ``["NC_…:g.123A>G", "NC_…:g.125T>C"]`` becomes + ``"NC_…:g.[123A>G;125T>C]"``. A single component is returned unchanged. + + Returns ``None`` when the components do not share a single accession and coordinate prefix — + they are then not expressible as one cis-phased block (e.g. members on different sequences). + """ + if not components: + return None + if len(components) == 1: + return components[0] + + accessions: set[str] = set() + prefixes: set[str] = set() + descriptions: list[str] = [] + for component in components: + accession, separator, remainder = component.partition(":") + match = _HGVS_COORD_PREFIX.match(remainder) + if not separator or not match: + return None + + accessions.add(accession) + prefixes.add(match.group(1)) + descriptions.append(match.group(2)) + + if len(accessions) != 1 or len(prefixes) != 1: + return None + + # Emit components in coordinate order so the combined string is deterministic regardless of + # member ordering. The VRS block digest is order-independent (so dedup is unaffected), but this + # string is surfaced in CSV export, where a stable, spec-conventional ordering is useful. + descriptions.sort(key=_cis_phased_sort_key) + return f"{accessions.pop()}:{prefixes.pop()}[{';'.join(descriptions)}]" diff --git a/src/mavedb/lib/identifiers.py b/src/mavedb/lib/identifiers.py index d5bfdbd46..7855b6af7 100644 --- a/src/mavedb/lib/identifiers.py +++ b/src/mavedb/lib/identifiers.py @@ -1,7 +1,10 @@ +import json +import logging import os from typing import Mapping, Optional, Union import eutils # type: ignore +import requests from eutils import EutilsNCBIError # type: ignore from eutils._internal.xmlfacades.pubmedarticle import PubmedArticle # type: ignore from eutils._internal.xmlfacades.pubmedarticleset import PubmedArticleSet # type: ignore @@ -23,6 +26,8 @@ from mavedb.models.uniprot_identifier import UniprotIdentifier from mavedb.models.uniprot_offset import UniprotOffset +logger = logging.getLogger(__name__) + # XXX these classes all have an "identifier" attribute but there's no superclass # to unify them ... @@ -313,6 +318,7 @@ async def find_generic_article( # When we are not provided a db name, we must try to match the provided identifier to a # publication from each one of our accepted databases. + last_fetch_error: Optional[Exception] = None for publication_db, identifier_valid in identifier_valid_for(identifier).items(): if identifier_valid: existing_publication = db.execute( @@ -322,7 +328,19 @@ async def find_generic_article( ).scalar_one_or_none() if not existing_publication: - external_publication = await db_specific_fetches[publication_db](identifier) + try: + external_publication = await db_specific_fetches[publication_db](identifier) + except (json.JSONDecodeError, requests.exceptions.RequestException) as exc: + logger.warning( + "Failed to fetch identifier %r from %s while fanning out over candidate databases.", + identifier, + publication_db, + exc_info=True, + ) + + external_publication = None + last_fetch_error = exc + found_articles[publication_db] = ( ExternalPublication(identifier, publication_db, external_publication) if external_publication @@ -331,6 +349,9 @@ async def find_generic_article( else: found_articles[publication_db] = existing_publication + if last_fetch_error is not None and not any(found_articles.values()): + raise last_fetch_error + return found_articles diff --git a/src/mavedb/lib/logging/context.py b/src/mavedb/lib/logging/context.py index 075efb586..8e05e7706 100644 --- a/src/mavedb/lib/logging/context.py +++ b/src/mavedb/lib/logging/context.py @@ -23,7 +23,10 @@ async def set_context(self, request: Union[Request, HTTPConnection]) -> dict: ctx: dict[str, Any] = {} ctx["request_ns"] = time.time_ns() - ctx["path"] = request.url.path + # The ASGI scope rather than request.url.path, which truncates at a '#'. Starlette builds + # request.url by re-parsing, so a variant URN's '#' opens a fragment there and the variant + # number and sub-resource are dropped: three /variants routes all logged the same path. + ctx["path"] = request.scope["path"] if isinstance(request, Request): ctx["method"] = request.method @@ -60,6 +63,12 @@ def save_to_logging_context(ctx: dict) -> dict: return context.data +def increment_logging_context(key: str, by: int = 1) -> None: + """Add ``by`` to a counter in the request's logging context. A no-op outside a request (worker, scripts).""" + if context.exists(): + context[key] = context.get(key, 0) + by + + def logging_context() -> dict: if not context.exists(): logger.debug("Could not access logging context. Context does not exist.") diff --git a/src/mavedb/lib/mapping/schema.py b/src/mavedb/lib/mapping/schema.py index 939b60401..02d377780 100644 --- a/src/mavedb/lib/mapping/schema.py +++ b/src/mavedb/lib/mapping/schema.py @@ -7,9 +7,31 @@ """ from datetime import datetime +from enum import Enum from typing import Any, Optional, TypedDict +class MappingOutcome(str, Enum): + """Per-record outcome stamped by dcd-mapping on every emitted score annotation. + + Mirrors ``dcd_mapping.schemas.MappingOutcome``. Lets a benign absence of a VRS allele + be told apart from a genuine failure -- a distinction ``error_message`` alone cannot + make (benign outcomes leave it ``None``). ``MAPPED`` is a success; ``INTRONIC`` and + ``NO_PROTEIN_CONSEQUENCE`` are benign skips (no allele, not failures); ``FAILED`` is the + only genuine failure. + """ + + MAPPED = "mapped" + INTRONIC = "intronic" + NO_PROTEIN_CONSEQUENCE = "no_protein_consequence" + FAILED = "failed" + + @property + def is_benign_absence(self) -> bool: + """True for outcomes that legitimately produce no allele yet are not failures.""" + return self in (MappingOutcome.INTRONIC, MappingOutcome.NO_PROTEIN_CONSEQUENCE) + + class GeneInfo(TypedDict, total=False): hgnc_symbol: Optional[str] selection_method: Optional[str] diff --git a/src/mavedb/lib/middleware/errors.py b/src/mavedb/lib/middleware/errors.py index 871eac5d9..b78c59525 100644 --- a/src/mavedb/lib/middleware/errors.py +++ b/src/mavedb/lib/middleware/errors.py @@ -3,6 +3,8 @@ import logging import time +from sqlalchemy.exc import DBAPIError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError from starlette.requests import Request from starlette.responses import JSONResponse from starlette.types import ASGIApp, Message, Receive, Scope, Send @@ -18,9 +20,25 @@ logger = logging.getLogger(__name__) +# SQLSTATEs Postgres raises when a query exceeds `statement_timeout` (query_canceled) or waits past +# `lock_timeout` (lock_not_available). +_DATABASE_TIMEOUT_SQLSTATES = frozenset({"57014", "55P03"}) +_BUSY_DETAIL = "The database is busy. Please try again shortly." + + +def _is_database_busy(err: BaseException) -> bool: + """Whether ``err`` means the database is too busy right now, rather than that the server is broken. + + Covers the server's statement and lock timeouts and a request that waited past ``pool_timeout`` for a + pooled connection. + """ + if isinstance(err, PoolTimeoutError): + return True + return isinstance(err, DBAPIError) and getattr(err.orig, "pgcode", None) in _DATABASE_TIMEOUT_SQLSTATES + class CatchAllErrorMiddleware: - """Turn an uncaught exception into a 500 that the browser is allowed to read. + """Turn an uncaught exception into a 500 (503 when the database is busy) that the browser is allowed to read. Starlette dispatches ``@app.exception_handler(Exception)`` from ``ServerErrorMiddleware``, which sits outside the user middleware stack. Its response therefore never passes through ``CORSMiddleware`` and @@ -61,14 +79,22 @@ async def send_wrapper(message: Message) -> None: save_to_logging_context(format_raised_exception_info_as_dict(err)) request = Request(scope, receive) + busy = _is_database_busy(err) response = JSONResponse( - status_code=500, - content={"detail": "Internal server error", "correlation_id": correlation_id_for_context()}, + status_code=503 if busy else 500, + content={ + "detail": _BUSY_DETAIL if busy else "Internal server error", + "correlation_id": correlation_id_for_context(), + }, ) try: - logger.error(msg="Uncaught exception.", extra=logging_context(), exc_info=err) - send_slack_error(err=err, request=request) + # No Slack alert: these arrive in bursts under load, and the request log counts them. + if busy: + logger.warning(msg="Database busy.", extra=logging_context(), exc_info=err) + else: + logger.error(msg="Uncaught exception.", extra=logging_context(), exc_info=err) + send_slack_error(err=err, request=request) finally: log_request(request, response, time.time_ns()) diff --git a/src/mavedb/lib/mondo.py b/src/mavedb/lib/mondo.py new file mode 100644 index 000000000..7906146c8 --- /dev/null +++ b/src/mavedb/lib/mondo.py @@ -0,0 +1,99 @@ +"""MONDO disease-term vocabulary: constants, MappableConcept serialization, and the generic term. + +Calibrations carry a disease/disorder context drawn from the Monarch Disease Ontology (MONDO). This +module owns the vocabulary's static surface: the generic "disease or disorder" root, conversion of a +stored :class:`MondoTerm` (or an OLS suggestion) to the GA4GH ``MappableConcept`` served on the wire, and +get-or-create of the generic term (mirroring :func:`mavedb.lib.taxonomies.find_or_create_taxonomy`). + +The generic term (``MONDO:0000001``) is the write-time default for a calibration with no specific +disease, so ``disease_term_id`` is never null. The annotation layer serializes whatever term a +calibration carries — generic or specific — as its VA-Spec disease condition (see +:func:`mavedb.lib.annotation.condition.calibration_disease_condition`). + +Typeahead search and OLS-backed validation of a submitted code live in :mod:`mavedb.lib.mondo_ols`, +kept separate because they need :mod:`mavedb.lib.logging.context` (an optional "server" dependency, +tracked for a core-compatible rework in +`#459 `_); this module has no such dependency, +so callers that only need the constants or serialization (e.g. the score calibration view model) stay +importable under core dependencies. +""" + +from typing import TypedDict + +from ga4gh.core.models import Coding, MappableConcept, iriReference +from sqlalchemy.orm import Session + +from mavedb.models.mondo_term import MondoTerm + +# The Monarch Disease Ontology (MONDO), the controlled vocabulary for disease terms used across MaveDB. +MONDO_SYSTEM = "https://purl.obolibrary.org/obo/mondo.owl" + +# The generic "disease or disorder" root, used when a calibration names no specific disease. +MONDO_GENERIC_CODE = "MONDO:0000001" +MONDO_GENERIC_LABEL = "disease or disorder" + + +class MondoSuggestion(TypedDict): + """A single MONDO term resolved from OLS: the CURIE code, its label, and its resolvable IRI.""" + + code: str + label: str + iri: str + + +def mondo_iri(code: str) -> str: + """The resolvable OBO IRI for a MONDO CURIE (e.g. ``MONDO:0015263`` → ``.../obo/MONDO_0015263``).""" + return f"https://purl.obolibrary.org/obo/{code.replace(':', '_')}" + + +def mondo_term_to_mappable_concept(term: MondoTerm) -> MappableConcept: + """Serialize a stored :class:`MondoTerm` as a GA4GH disease ``MappableConcept``.""" + return MappableConcept( + conceptType="Disease", + name=term.label, + primaryCoding=Coding( + code=term.code, + system=term.system, + systemVersion=term.system_version, + iris=[iriReference(root=mondo_iri(term.code))], + ), + ) + + +def mondo_suggestion_to_mappable_concept(suggestion: MondoSuggestion) -> MappableConcept: + """Build a disease ``MappableConcept`` from an OLS search suggestion (for typeahead results).""" + return MappableConcept( + conceptType="Disease", + name=suggestion["label"], + primaryCoding=Coding( + code=suggestion["code"], + system=MONDO_SYSTEM, + iris=[iriReference(root=suggestion["iri"])], + ), + ) + + +def generic_disease_mappable_concept() -> MappableConcept: + """The generic "disease or disorder" concept, MaveDB's single unspecified-disease sentinel.""" + return MappableConcept( + conceptType="Disease", + name=MONDO_GENERIC_LABEL, + primaryCoding=Coding( + code=MONDO_GENERIC_CODE, + system=MONDO_SYSTEM, + iris=[iriReference(root=mondo_iri(MONDO_GENERIC_CODE))], + ), + ) + + +def get_generic_disease_term(db: Session) -> MondoTerm: + """Get-or-create the generic disease term. Known-canonical, so it needs no OLS round trip.""" + term = ( + db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == MONDO_GENERIC_CODE).one_or_none() + ) + if term is None: + term = MondoTerm(code=MONDO_GENERIC_CODE, system=MONDO_SYSTEM, label=MONDO_GENERIC_LABEL) # type: ignore[call-arg] + db.add(term) + db.flush() + + return term diff --git a/src/mavedb/lib/mondo_ols.py b/src/mavedb/lib/mondo_ols.py new file mode 100644 index 000000000..c6a58bcec --- /dev/null +++ b/src/mavedb/lib/mondo_ols.py @@ -0,0 +1,138 @@ +"""MONDO disease-term search and validation against the EBI Ontology Lookup Service (OLS4). + +Split out from :mod:`mavedb.lib.mondo`: this module resolves and validates a submitted MONDO code +against the live OLS4 service, which needs :mod:`mavedb.lib.logging.context` for structured request +logging — and that, in turn, needs ``starlette``, an optional "server" dependency (see +`#459 `_ for making logging context available to +core dependencies directly). Keeping the OLS calls here, separate from :mod:`mavedb.lib.mondo`'s +constants and serialization, lets callers that only need the static vocabulary stay importable without +the server extras. +""" + +import logging +from typing import Any, Optional + +import httpx +from sqlalchemy.orm import Session + +from mavedb.lib.exceptions import MondoServiceError +from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context +from mavedb.lib.mondo import MONDO_GENERIC_CODE, MONDO_SYSTEM, MondoSuggestion, get_generic_disease_term, mondo_iri +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.mondo_term import MondoTerm + +logger = logging.getLogger(__name__) + +# EBI Ontology Lookup Service (OLS4) — the source for MONDO term search and validation. +OLS_BASE_URL = "https://www.ebi.ac.uk/ols4/api" +MONDO_ONTOLOGY = "mondo" +# OLS can be slow; the default httpx 5s timeout is too tight for its autocomplete on short queries. +OLS_TIMEOUT = 10.0 + +DEFAULT_SEARCH_LIMIT = 20 + + +async def search_mondo(query: str, limit: int = DEFAULT_SEARCH_LIMIT) -> list[MondoSuggestion]: + """Search MONDO disease classes by free text via OLS, for the disease typeahead.""" + if not query or not query.strip(): + return [] + + # OLS4's /select endpoint is the autocomplete-optimized path; /search times out on broad prefixes + # (e.g. a two-letter query), which is often what a typeahead sends. + params: dict[str, Any] = { + "q": query.strip(), + "ontology": MONDO_ONTOLOGY, + "rows": limit, + "fieldList": "iri,label,obo_id", + } + save_to_logging_context({"mondo_search_query": query, "mondo_search_limit": limit}) + try: + async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: + response = await client.get(f"{OLS_BASE_URL}/select", params=params) + response.raise_for_status() + docs = response.json().get("response", {}).get("docs", []) + except httpx.HTTPError as exc: + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + logger.error(msg="MONDO/OLS search request failed.", exc_info=exc, extra=logging_context()) + raise MondoServiceError("Disease ontology service temporarily unavailable") from exc + + return [ + MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(doc["obo_id"]))) + for doc in docs + if doc.get("obo_id") and doc.get("label") + ] + + +async def fetch_mondo_term(code: str) -> Optional[MondoSuggestion]: + """Resolve one MONDO CURIE to its canonical label + IRI via OLS, or None if it does not exist.""" + params: dict[str, Any] = { + "q": code, + "ontology": MONDO_ONTOLOGY, + "queryFields": "obo_id", + "exact": "true", + "rows": 1, + "fieldList": "iri,label,obo_id", + } + try: + async with httpx.AsyncClient(timeout=OLS_TIMEOUT) as client: + response = await client.get(f"{OLS_BASE_URL}/search", params=params) + response.raise_for_status() + docs = response.json().get("response", {}).get("docs", []) + except httpx.HTTPError as exc: + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + logger.error(msg="MONDO/OLS term lookup failed.", exc_info=exc, extra=logging_context()) + raise MondoServiceError("Disease ontology service temporarily unavailable") from exc + + if not docs: + return None + + doc = docs[0] + return MondoSuggestion(code=doc["obo_id"], label=doc["label"], iri=doc.get("iri", mondo_iri(code))) + + +async def find_or_create_mondo_term(db: Session, code: str) -> MondoTerm: + """Find an existing MONDO term by code, or create one after validating it against OLS. + + A new code is validated against OLS and its canonical label stored. An unknown code raises + ``ValidationError``; an OLS outage raises ``MondoServiceError`` (via :func:`fetch_mondo_term`) so a + term is never persisted without a validated label — a controlled vocabulary must stay validated. + """ + # The generic term is canonical and seeded — resolve it directly, never via OLS. + if code == MONDO_GENERIC_CODE: + return get_generic_disease_term(db) + + term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == code).one_or_none() + if term is not None: + return term + + resolved = await fetch_mondo_term(code) + if resolved is None: + raise ValidationError(f"'{code}' is not a valid MONDO disease term.", custom_loc=["body", "disease"]) + + # OLS may normalize the submitted code (case, or an alias/obsolete id) to a different canonical code, + # so re-check existence by the resolved code before inserting: keying the lookup on the same value we + # insert under keeps this idempotent and avoids a UNIQUE(system, code) violation when the canonical + # row already exists under a different submitted spelling. + canonical_code = resolved["code"] + term = db.query(MondoTerm).filter(MondoTerm.system == MONDO_SYSTEM, MondoTerm.code == canonical_code).one_or_none() + if term is not None: + return term + + term = MondoTerm(code=canonical_code, system=MONDO_SYSTEM, label=resolved["label"]) # type: ignore[call-arg] + db.add(term) + db.flush() + + return term + + +async def resolve_disease_term(db: Session, code: Optional[str]) -> MondoTerm: + """Resolve a calibration's submitted disease code to a stored MONDO term. + + ``None`` (no disease specified) resolves to the generic "disease or disorder" term; any other code + is validated/created via :func:`find_or_create_mondo_term`. Shared by the score calibration create + and update paths so the write-time default lives in one place. + """ + if code is None: + return get_generic_disease_term(db) + + return await find_or_create_mondo_term(db, code) diff --git a/src/mavedb/lib/permissions/score_set.py b/src/mavedb/lib/permissions/score_set.py index ba35eaf91..cb61a6ad6 100644 --- a/src/mavedb/lib/permissions/score_set.py +++ b/src/mavedb/lib/permissions/score_set.py @@ -234,6 +234,12 @@ def _handle_set_scores_action( Only owners, contributors, and admins can modify the scores data within a ScoreSet. This is a critical operation that affects the scientific data. + Publishing freezes that data: a published score set's scores are the record other work cites, and + re-uploading them would silently change results already in circulation, along with anything + derived from the variants they create (mappings, and the bin membership of any calibration built + on the score set). Corrections to published data go through a superseding score set instead. + Admins keep the ability as an operational escape hatch, mirroring DELETE. + Args: user_data: The user's authentication data. entity: The ScoreSet entity whose scores are being modified. @@ -246,12 +252,12 @@ def _handle_set_scores_action( PermissionResponse: Permission result with appropriate HTTP status. """ ## Allow set scores access under the following conditions: - # The owner or contributors may set scores. - if user_is_owner or user_is_contributor: - return PermissionResponse(True) - # Users with these specific roles may set scores. + # Users with these specific roles may set scores on any score set. if roles_permitted(active_roles, [UserRole.admin]): return PermissionResponse(True) + # The owner or contributors may set scores only while the score set is still private. + if (user_is_owner or user_is_contributor) and private: + return PermissionResponse(True) return deny_action_for_entity(entity, private, user_data, user_is_contributor or user_is_owner, "score set") diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 1796b3a8d..afa4723ac 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -1,7 +1,10 @@ """Utilities for building and mutating score calibration ORM objects.""" +import logging import math -from typing import Optional, Union +from collections import Counter +from datetime import date +from typing import Optional, Sequence, Union, cast import pandas as pd from sqlalchemy import Float, and_, select @@ -9,23 +12,252 @@ from mavedb.lib.acmg import find_or_create_acmg_classification from mavedb.lib.identifiers import find_or_create_publication_identifier -from mavedb.lib.types.score_calibrations import ClassificationDict +from mavedb.lib.mondo_ols import resolve_disease_term +from mavedb.lib.types.score_calibrations import ( + CalibrationControlSnapshot, + CalibrationVariantLinkSnapshot, + CalibrationVariantRelinkReport, + ClassificationDict, + VariantIdentity, +) from mavedb.lib.validation.constants.general import ( calibration_class_column_name, calibration_variant_column_name, hgvs_nt_column, hgvs_pro_column, ) +from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.validation.utilities import inf_or_float +from mavedb.lib.variants import score_from_variant_data +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) from mavedb.models.score_calibration_publication_identifier import ScoreCalibrationPublicationIdentifierAssociation from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.models.variant import Variant from mavedb.view_models import score_calibration +logger = logging.getLogger(__name__) + + +def _fetch_control_variants_by_urn( + db: Session, + score_set: ScoreSet, + controls: Sequence[score_calibration.CalibrationControlCreate], +) -> dict[str, Variant]: + """Fetch the ``Variant`` rows for a submission's control URNs, scoped to the score set, keyed by URN. + + A single scoped query serves both membership validation (a URN absent from the result does not + belong to the score set) and row construction, so validating and building a submission share one + lookup rather than each issuing its own. + """ + submitted_urns = [control.variant_urn for control in controls] + return { + variant.urn: variant + for variant in db.scalars( + select(Variant).where(Variant.score_set_id == score_set.id, Variant.urn.in_(submitted_urns)) + ).all() + # Every returned row matched an IN over the submitted (non-null) URNs, so its urn is non-null; + # the guard both states that and narrows the key type from Optional[str]. + if variant.urn is not None + } + + +def _validate_control_submission( + controls: Sequence[score_calibration.CalibrationControlCreate], + variants_by_urn: dict[str, Variant], +) -> None: + """Duplicate and score-set-membership checks for a control submission, given its resolved variants. + + A control earns its role as calibration evidence from a variant the assay actually scored, so each + must reference a variant belonging to the calibration's own score set. Duplicate URNs within one + submission are caught here with a readable message rather than deferred to the + ``UNIQUE(calibration_id, variant_id)`` database constraint. + + Raises: + ValidationError: If any control URN is duplicated within the submission, absent from MaveDB, + or belongs to a different score set. + """ + counts = Counter(control.variant_urn for control in controls) + + duplicate_urns = {urn for urn, count in counts.items() if count > 1} + if duplicate_urns: + raise ValidationError( + f"Duplicate control variant URNs detected within the submission: {', '.join(sorted(duplicate_urns))}." + ) + + missing_urns = counts.keys() - variants_by_urn.keys() + if missing_urns: + raise ValidationError( + "The following control variants do not belong to the calibration's score set: " + f"{', '.join(sorted(missing_urns))}." + ) + + +def validate_calibration_controls_in_score_set( + db: Session, + score_set: ScoreSet, + controls: Optional[Sequence[score_calibration.CalibrationControlCreate]], +) -> list[str]: + """Validate calibration controls against a score set and return their variant URNs. + + A thin wrapper over :func:`_fetch_control_variants_by_urn` and :func:`_validate_control_submission` + for callers that only need the validated URNs; :func:`build_calibration_controls` shares the same + two steps to avoid re-fetching the variants. + + Args: + db: Database session used for the lookup. + score_set: The score set the calibration belongs to. + controls: Controls submitted on create or modify. ``None`` or empty returns ``[]``. + + Returns: + The submitted and validated variant URNs, in order; empty for ``None``/empty input. + + Raises: + ValidationError: If any control URN is duplicated within the submission, absent from + MaveDB, or belongs to a different score set. + """ + if not controls: + return [] + + variants_by_urn = _fetch_control_variants_by_urn(db, score_set, controls) + _validate_control_submission(controls, variants_by_urn) + return [control.variant_urn for control in controls] + + +def require_phi_affirmation(has_controls: bool, controls_not_phi: Optional[bool]) -> None: + """Reject a calibration state that would store controls without a PHI affirmation. + + A calibration may hold controls only while ``controls_not_phi`` is ``True``; ``None`` (unaddressed) + and ``False`` (declined) both block. Call this before staging any write that sets either side. + + Raises: + ValidationError: If ``has_controls`` and the affirmation is not ``True``. + """ + if has_controls and controls_not_phi is not True: + raise ValidationError( + "Calibration controls cannot be saved until they are affirmed to be free of protected health " + "information (PHI). Set controls_not_phi to true.", + custom_loc=["body", "controlsNotPhi"], + ) + + +def build_calibration_controls( + db: Session, + score_set: ScoreSet, + controls: Optional[Sequence[score_calibration.CalibrationControlCreate]], + user: User, +) -> list[CalibrationControl]: + """Validate and construct transient ``CalibrationControl`` rows for a calibration. + + Fetches the submission's ``Variant`` rows once, validates them (duplicates + score-set membership, + see :func:`_validate_control_submission`), then turns each control into an unattached + ``CalibrationControl`` with audit fields set. The single fetch is reused for validation and + construction, so this issues one variant query rather than one to validate and another to build. + The caller assigns the returned rows to the calibration's ``controls`` collection and commits. + Returns an empty list for ``None`` or empty input. + + Args: + db: Database session. + score_set: The score set the calibration belongs to; controls resolve within it. + controls: Submitted controls, or ``None``/empty for none. + user: The acting user, recorded on each control's audit fields. + """ + if not controls: + return [] + + variants_by_urn = _fetch_control_variants_by_urn(db, score_set, controls) + _validate_control_submission(controls, variants_by_urn) + + return [ + CalibrationControl( + variant=variants_by_urn[control.variant_urn], + clinical_status=control.clinical_status, + created_by=user, + modified_by=user, + ) + for control in controls + ] + + +def calibration_preference_key(calibration: ScoreCalibration) -> tuple[int, int, int]: + """The UI's default-calibration preference cascade as a sort key (lower sorts first = more preferred): + ``primary``, then ``investigator_provided``, then non-``research_use_only``. Mirrors the UI's + ``activeCalibrationOptions`` default selection, so the API and UI agree on which calibration is "the + default" for a score set. + + Intentionally stops at the cascade tiers and carries **no** evidence strength (a variant-specific + concern) and **no** id — callers append their own deterministic tiebreak (calibration id or urn), and, where + a within-tier strongest-evidence tiebreak is wanted, :func:`classification_evidence_strength`. + """ + return ( + 0 if calibration.primary else 1, + 0 if calibration.investigator_provided else 1, + 0 if not calibration.research_use_only else 1, + ) + + +def classification_evidence_strength( + classification: Optional[ScoreCalibrationFunctionalClassification], +) -> tuple[float, int]: + """``(magnitude, direction)`` of a functional classification's evidence, for ranking. + + ``magnitude`` is the evidence strength (larger = stronger); ``direction`` biases toward pathogenic + (0 = pathogenic, 1 = benign, 2 = unknown). ACMG points are the primary signal (signed: + positive = pathogenic), falling back to ``|ln(oddspaths_ratio)|`` then the functional call + (abnormal → pathogenic, normal → benign) for direction only. + """ + if classification is None: + return (0.0, 2) + + acmg = classification.acmg_classification + if acmg is not None and acmg.points is not None: + points = float(acmg.points) + return (abs(points), 0 if points > 0 else 1 if points < 0 else 2) + + # OddsPath is a ratio centered at 1 (pathogenic >1, benign <1); ln maps it to a symmetric log-odds + # strength around 0 (x and 1/x are equal-and-opposite), comparable across direction and commensurate + # with the already-log-scaled ACMG points. + ratio = classification.oddspaths_ratio + if ratio is not None and ratio > 0: + return (abs(math.log(ratio)), 0 if ratio > 1 else 1 if ratio < 1 else 2) + + if classification.functional_classification == FunctionalClassification.abnormal: + return (0.0, 0) + elif classification.functional_classification == FunctionalClassification.normal: + return (0.0, 1) + + return (0.0, 2) + + +def classifications_by_variant( + db: Session, variant_ids: Sequence[int] +) -> dict[int, list[tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]]]: + """Each variant's functional classifications with their calibrations, in one query for all of + ``variant_ids``. Unordered and unfiltered: callers apply their own visibility rule and preference.""" + if not variant_ids: + return {} + + association = score_calibration_functional_classification_variants_association_table + rows = db.execute( + select(association.c.variant_id, ScoreCalibrationFunctionalClassification, ScoreCalibration) + .join(association, association.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(association.c.variant_id.in_(variant_ids)) + ).tuples() + + by_variant: dict[int, list[tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]]] = {} + for variant_id, classification, calibration in rows: + by_variant.setdefault(variant_id, []).append((classification, calibration)) + + return by_variant + def create_functional_classification( db: Session, @@ -181,6 +413,8 @@ async def _create_score_calibration( by_alias=False, exclude={ "functional_classifications", + "controls", + "disease", "threshold_sources", "evidence_sources", "method_sources", @@ -193,6 +427,8 @@ async def _create_score_calibration( modified_by=user, ) # type: ignore[call-arg] + calibration.disease_term = await resolve_disease_term(db, getattr(calibration_create, "disease", None)) + if containing_score_set: calibration.score_set = containing_score_set calibration.score_set_id = containing_score_set.id @@ -250,6 +486,9 @@ async def create_score_calibration_in_score_set( if not calibration_create.score_set_urn: raise ValueError("score_set_urn must be provided to create a score calibration within a score set.") + submitted_controls = getattr(calibration_create, "controls", None) + require_phi_affirmation(bool(submitted_controls), getattr(calibration_create, "controls_not_phi", None)) + containing_score_set = db.query(ScoreSet).where(ScoreSet.urn == calibration_create.score_set_urn).one() calibration = await _create_score_calibration(db, calibration_create, user, variant_classes, containing_score_set) @@ -261,6 +500,8 @@ async def create_score_calibration_in_score_set( else: calibration.investigator_provided = False + calibration.controls = build_calibration_controls(db, containing_score_set, submitted_controls, user) + db.add(calibration) return calibration @@ -413,6 +654,24 @@ async def modify_score_calibration( containing_score_set = db.query(ScoreSet).where(ScoreSet.urn == calibration_update.score_set_urn).one() + # Replace semantics: a provided controls list (even empty) replaces all existing controls, while None + # leaves them untouched. Re-acknowledgment: an explicit controls_not_phi in the request always wins; + # otherwise replacing the controls invalidates the prior affirmation, while leaving them preserves it. + # Resolved up front so a rejected update stages nothing. + submitted_controls = getattr(calibration_update, "controls", None) + if "controls_not_phi" in calibration_update.model_fields_set: + controls_not_phi = calibration_update.controls_not_phi + elif submitted_controls is not None: + controls_not_phi = None + else: + controls_not_phi = calibration.controls_not_phi + + # Withdrawing the affirmation removes the controls stored under it. + if controls_not_phi is not True and submitted_controls is None: + submitted_controls = [] + + require_phi_affirmation(bool(submitted_controls), controls_not_phi) + relation_sources = ( (ScoreCalibrationRelation.threshold, calibration_update.threshold_sources or []), (ScoreCalibrationRelation.evidence, calibration_update.evidence_sources or []), @@ -459,6 +718,11 @@ async def modify_score_calibration( for attr, value in calibration_update.model_dump().items(): if attr not in { "functional_classifications", + "controls", + "disease", + # controls_not_phi carries re-acknowledgment semantics; set it explicitly so an + # unrelated edit cannot silently wipe a prior affirmation. + "controls_not_phi", "threshold_sources", "evidence_sources", "method_sources", @@ -470,6 +734,8 @@ async def modify_score_calibration( }: setattr(calibration, attr, value) + calibration.disease_term = await resolve_disease_term(db, getattr(calibration_update, "disease", None)) + calibration.score_set = containing_score_set calibration.score_set_id = containing_score_set.id calibration.publication_identifier_associations = updated_assocs @@ -482,6 +748,15 @@ async def modify_score_calibration( db.add(persisted_functional_range) calibration.functional_classifications.append(persisted_functional_range) + if submitted_controls is not None: + for control in list(calibration.controls): + db.delete(control) + calibration.controls.clear() + db.flush() + calibration.controls = build_calibration_controls(db, containing_score_set, submitted_controls, user) + + calibration.controls_not_phi = controls_not_phi + db.add(calibration) return calibration @@ -777,18 +1052,8 @@ def variants_for_functional_classification( continue elif functional_classification.range is not None and len(functional_classification.range) == 2: - try: - container = v.data.get("score_data") if isinstance(v.data, dict) else None - if not container or not isinstance(container, dict): - continue - - raw = container.get("score") - if raw is None: - continue - - score = float(raw) - - except Exception: # noqa: BLE001 + score = score_from_variant_data(v.data) + if score is None: continue if functional_classification.score_is_contained_in_range(score): @@ -833,3 +1098,235 @@ def variant_classification_df_to_dict( classifications[functional_class].add(index_element) return {"indexed_by": index_column, "classifications": classifications} + + +def snapshot_calibration_variant_links(db: Session, score_set: ScoreSet) -> list[CalibrationVariantLinkSnapshot]: + """Record the calibration variant references a re-upload cannot reconstruct on its own. + + Re-uploading a score set's data deletes and recreates all of its ``Variant`` rows, which breaks + the foreign keys calibrations hold into them: ``calibration_controls.variant_id`` and the + functional-classification membership association. Neither carries an ``ON DELETE`` action, on + purpose — the resulting ``RESTRICT`` protects hand-entered controls from being destroyed by an + unrelated delete. Capturing the irreproducible references as :data:`VariantIdentity` tuples before + the delete lets :func:`restore_calibration_variant_links` re-point the survivors afterwards. + Variant URNs cannot serve as the key because they are renumbered on every upload. + + The rule for what to capture is whether the depositor asserted the reference or MaveDB derived it: + + * **Controls** carry clinical significance sourced from outside MaveDB, submitted inline or as a + ``controls_file``. Nothing in a score upload can regenerate them, so they are remembered. + * **Class-based bin membership** comes from the ``classes_file`` required of a class-based + calibration. That file is not part of a score upload either, so it too survives only by identity. + * **Range-based bin membership** is derived — purely a function of the variants' scores. It is + left out and recomputed from the new upload instead, which avoids holding an entry per variant + in memory and keeps membership honest when a re-upload moves a score across a threshold. + + The queries scope to variants in ``score_set``, which keeps the snapshot aligned with exactly the + rows the caller deletes. Every reference is already so scoped — controls are validated against the + calibration's own score set and bin membership is only ever populated from it — so the condition + holds the two halves together rather than filtering anything out. Were they to diverge, restore + would re-create a row whose original was never deleted and trip the unique constraint on + ``(calibration_id, variant_id)``. + + This function only reads. The caller performs the deletes. + + Args: + db: Database session. + score_set: The score set whose variants are about to be replaced. + + Returns: + One snapshot per calibration holding at least one irreproducible reference; empty when no + calibration has one. A score set whose calibrations use only range-based bins yields an empty + list and still needs :func:`restore_calibration_variant_links` called to re-bin them. + """ + calibration_ids = [calibration.id for calibration in score_set.score_calibrations] + if not calibration_ids: + return [] + + snapshots: dict[int, CalibrationVariantLinkSnapshot] = {} + + control_rows = db.execute( + select( + CalibrationControl.calibration_id, + CalibrationControl.clinical_status, + CalibrationControl.created_by_id, + CalibrationControl.creation_date, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + ) + .join(Variant, Variant.id == CalibrationControl.variant_id) + .where(CalibrationControl.calibration_id.in_(calibration_ids), Variant.score_set_id == score_set.id) + ).all() + + for calibration_id, clinical_status, created_by_id, creation_date, hgvs_nt, hgvs_pro, hgvs_splice in control_rows: + snapshot = snapshots.setdefault(calibration_id, CalibrationVariantLinkSnapshot(calibration_id=calibration_id)) + snapshot.controls.append( + CalibrationControlSnapshot( + identity=(hgvs_nt, hgvs_pro, hgvs_splice), + clinical_status=clinical_status, + created_by_id=created_by_id, + creation_date=creation_date, + ) + ) + + association = score_calibration_functional_classification_variants_association_table + membership_rows = db.execute( + select( + ScoreCalibrationFunctionalClassification.calibration_id, + ScoreCalibrationFunctionalClassification.id, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + ) + .join(association, association.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id) + .join(Variant, Variant.id == association.c.variant_id) + .where( + ScoreCalibrationFunctionalClassification.calibration_id.in_(calibration_ids), + ScoreCalibrationFunctionalClassification.class_.is_not(None), + Variant.score_set_id == score_set.id, + ) + ).all() + + for calibration_id, classification_id, hgvs_nt, hgvs_pro, hgvs_splice in membership_rows: + snapshot = snapshots.setdefault(calibration_id, CalibrationVariantLinkSnapshot(calibration_id=calibration_id)) + snapshot.classification_members.setdefault(classification_id, []).append((hgvs_nt, hgvs_pro, hgvs_splice)) + + return list(snapshots.values()) + + +def restore_calibration_variant_links( + db: Session, + score_set: ScoreSet, + snapshots: Sequence[CalibrationVariantLinkSnapshot], + updater: User, +) -> CalibrationVariantRelinkReport: + """Re-establish a score set's calibration variant references after its variants are recreated. + + Each calibration is handled by the two mechanisms its references call for: + + * **Controls and class-based bin membership** are carried across by + :data:`VariantIdentity`. A reference whose identity is absent from the new upload describes a + variant the assay no longer scores, so it is dropped rather than guessed at; an identity + matching more than one new variant is ambiguous and dropped for the same reason. Dropping + rather than failing is deliberate: a re-upload that aborted on a vanished control would leave + depositors unable to correct their own data. + * **Range-based bin membership** is recomputed from the new scores via + :func:`variants_for_functional_classification`, the same helper that populated it originally. + Carrying the old membership across might leave a variant filed under a range its new score no + longer falls in, so the recorded membership would simply be wrong. + + A classification always has exactly one of ``range`` or ``class_`` set (enforced by the view + models), so every classification falls squarely into one branch or the other. + + ``controls_not_phi`` is left as it was. The relinked controls are a subset of the set the + submitter affirmed, so the affirmation still covers them, and controls may only be stored while it + holds (see :func:`require_phi_affirmation`). + + Changes are staged on the session; the caller commits. + + Args: + db: Database session. + score_set: The score set whose variants have just been recreated. + snapshots: Output of :func:`snapshot_calibration_variant_links`, taken before the delete. May + be empty while calibrations still need re-binning. + updater: The user who triggered the re-upload, recorded on the rows this relink touches. + + Returns: + Counts of relinked, dropped and re-binned references, for the caller's job log. + """ + report = CalibrationVariantRelinkReport() + if not score_set.score_calibrations: + return report + + # Only controls and class-based bin membership are re-resolved by identity; range-based bins are + # recomputed from the new scores below and never touch this map. Building it materializes every + # variant of the score set, so skip that entirely when nothing needs identity resolution — the + # common case of a re-upload whose calibrations use only range-based bins. + variants_by_identity: dict[VariantIdentity, Optional[Variant]] = {} + if any(snapshot.controls or snapshot.classification_members for snapshot in snapshots): + for new_variant in db.scalars(select(Variant).where(Variant.score_set_id == score_set.id)).all(): + identity = (new_variant.hgvs_nt, new_variant.hgvs_pro, new_variant.hgvs_splice) + # Upstream validation rejects duplicate variants, but a collision here would otherwise bind + # every reference to whichever row happened to be seen first. Mark it unresolvable instead. + variants_by_identity[identity] = None if identity in variants_by_identity else new_variant + + snapshots_by_calibration_id = {snapshot.calibration_id: snapshot for snapshot in snapshots} + + for calibration in score_set.score_calibrations: + calibration_id = cast(int, calibration.id) + snapshot = snapshots_by_calibration_id.get( + calibration_id, CalibrationVariantLinkSnapshot(calibration_id=calibration_id) + ) + + # The linkage rows were removed with Core deletes, which leave any loaded collections holding + # rows that no longer exist. Expire them so the ORM rebuilds from the post-delete state rather + # than issuing deletes for rows that are already gone. + db.expire(calibration, ["controls"]) + + dropped_controls = 0 + relinked_controls: list[CalibrationControl] = [] + for control in snapshot.controls: + control_variant = variants_by_identity.get(control.identity) + if control_variant is None: + # Dropping leaves controls_not_phi untouched; the survivors are still covered by it. + dropped_controls += 1 + continue + + relinked_controls.append( + CalibrationControl( + calibration_id=calibration.id, + variant_id=control_variant.id, + clinical_status=control.clinical_status, + created_by_id=control.created_by_id, + creation_date=control.creation_date, + modified_by_id=updater.id, + modification_date=date.today(), + ) + ) + + db.add_all(relinked_controls) + report.controls_relinked += len(relinked_controls) + report.controls_dropped += dropped_controls + + dropped_members = 0 + for classification in calibration.functional_classifications: + db.expire(classification, ["variants"]) + + if classification.class_ is not None: + members: list[Variant] = [] + for identity in snapshot.classification_members.get(cast(int, classification.id), []): + member_variant = variants_by_identity.get(identity) + if member_variant is None: + dropped_members += 1 + continue + + members.append(member_variant) + + report.classification_members_relinked += len(members) + else: + members = variants_for_functional_classification(db, classification, use_sql=True) + report.classifications_rebinned += 1 + report.classification_members_rebinned += len(members) + + classification.variants = members + db.add(classification) + + report.classification_members_dropped += dropped_members + + # Re-binning is the expected mechanical consequence of new scores, so it does not count as an + # edit. Losing a hand-entered reference does. + if dropped_controls or dropped_members: + calibration.modified_by = updater + db.add(calibration) + + if report.controls_dropped or report.classification_members_dropped: + logger.warning( + "Dropped %s calibration control(s) and %s class-based bin membership(s) from score set %s: their " + "variants are absent from the new upload.", + report.controls_dropped, + report.classification_members_dropped, + score_set.urn, + ) + + return report diff --git a/src/mavedb/lib/score_distribution.py b/src/mavedb/lib/score_distribution.py new file mode 100644 index 000000000..2c784c3a5 --- /dev/null +++ b/src/mavedb/lib/score_distribution.py @@ -0,0 +1,62 @@ +"""Precomputed summaries of a score set's score distribution. + +The variant page draws a small histogram of each measured score set beside the variant's score. Fetching +every variant to draw it is too slow for large score sets, so the summary is computed once when scores are +written and stored on the score set (``ScoreSet.score_distribution``). +""" + +import math +from typing import Any, Iterable, Optional + +from mavedb.lib.variants import score_from_variant_data + +SCORE_DISTRIBUTION_VERSION = 1 +"""Bump when the binning changes, then re-run the backfill with ``--force``.""" + +SCORE_DISTRIBUTION_BIN_COUNT = 24 + + +def summarize_scores(scores: Iterable[Optional[float]]) -> Optional[dict[str, Any]]: + """Bin scores into equal-width bins spanning their full range. + + The full ``[min, max]`` is used rather than a clipped range: percentile clipping always hides some + scores, and IQR fences drop a whole mode in bimodal genome-editing data. + + Args: + scores: Scores to summarize. ``None`` and non-finite entries are counted in ``null_count`` and not binned. + + Returns: + ``{"version", "min", "max", "counts", "null_count"}``, or ``None`` when there are no numeric scores. + The maximum falls in the last bin. When every score is the same value, all of them fall in the + first bin. + """ + values: list[float] = [] + null_count = 0 + for score in scores: + if score is None or not math.isfinite(score): + null_count += 1 + else: + values.append(score) + + if not values: + return None + + low, high = min(values), max(values) + counts = [0] * SCORE_DISTRIBUTION_BIN_COUNT + width = (high - low) / SCORE_DISTRIBUTION_BIN_COUNT + for value in values: + index = 0 if width == 0 else min(int((value - low) / width), SCORE_DISTRIBUTION_BIN_COUNT - 1) + counts[index] += 1 + + return { + "version": SCORE_DISTRIBUTION_VERSION, + "min": low, + "max": high, + "counts": counts, + "null_count": null_count, + } + + +def summarize_variant_data(variant_data: Iterable[Optional[dict[str, Any]]]) -> Optional[dict[str, Any]]: + """Summarize scores read from variants' ``data`` JSONB. See :func:`summarize_scores`.""" + return summarize_scores(score_from_variant_data(data) for data in variant_data) diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py new file mode 100644 index 000000000..bb1952608 --- /dev/null +++ b/src/mavedb/lib/score_set_variants.py @@ -0,0 +1,220 @@ +"""Lean whole-set view backing the score-set page (table, heatmap, score/effect histograms). + +Assembles one record per variant in a single query: the score set's variants, with each per-variant +annotation fetched by a ``LATERAL ... LIMIT 1`` lookup (see :func:`get_lean_score_set_variants` for why). + +Each record carries the **submitted** HGVS (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the depositor's +target frame) and the **mapped** (reference-frame) HGVS as a ``MappedTriple`` — one canonical HGVS per +level (``genomic``/``cdna``/``protein``), with ``assay_level`` naming the measured slot. A nucleotide +assay fills all three slots; a protein assay fills only ``protein`` (the c/g fan-out is ambiguous, so +no pick is fabricated). This is the canonical projection only, never the reverse-translation fan-out. + +The mapped slots come from the record's measured projection group (the group its authoritative link points +at), which describes the measured variant at every level it has, including levels with no VRS allele +(an intronic coding projection). Where the group is not yet written, only the measured slot is filled. +Does not assemble Cat-VRS. + +``as_of`` reconstructs the annotation layer at a past instant; submitted HGVS and scores are immutable +and unaffected. Defaults to currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +from sqlalchemy import and_, select, true +from sqlalchemy.orm import Session, aliased + +from mavedb.lib.hgvs import parse_simple_substitution +from mavedb.lib.variants import score_from_variant_data +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class HgvsField: + """An HGVS string plus its parsed position/ref/alt when it's a placeable simple substitution. + + ``position``/``ref``/``alt`` are ``None`` for splice/intronic, indels, frameshifts, and + multivariants — those carry ``hgvs`` alone. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + +@dataclass(frozen=True) +class MappedTriple: + """The canonical mapped (reference-frame) HGVS at each level, or ``None`` where absent. + + A nucleotide assay populates the measured slot and whichever projections its measured projection + group holds (no protein for an intronic or UTR change); a protein assay populates only ``protein`` — + the c/g fan-out is ambiguous, so no pick is fabricated for ``cdna``/``genomic``. + + ``cdna`` is the level-invariant field: populated whether the assay was measured at ``cdna`` or + ``genomic``, so it's the field to key cross-assay searches on. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + +@dataclass(frozen=True) +class LeanVariantRecord: + """One pre-chewed per-variant record, keyed by ``variant_urn``. + + ``consequence`` is the allele's VEP molecular-consequence call. ``clingen_allele_id``/``assay_level_digest`` + bridge into the annotation dimensions. ``mapped`` is the canonical :class:`MappedTriple`; + ``assay_level`` names which of its slots is the measured one. Any field is ``None`` when its + source is absent (an unmapped variant → null ``assay_level`` and an empty triple). + """ + + variant_urn: str + score: Optional[float] + consequence: Optional[str] + clingen_allele_id: Optional[str] + assay_level_digest: Optional[str] + hgvs_nt: Optional[HgvsField] + hgvs_pro: Optional[HgvsField] + hgvs_splice: Optional[HgvsField] + assay_level: Optional[SequenceLevel] + mapped: MappedTriple + + +def _level(value: Optional[str]) -> Optional[SequenceLevel]: + """Coerce a stored assay-level string into :class:`SequenceLevel`, or ``None`` if absent.""" + return SequenceLevel(value) if value else None + + +def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: + """Wrap an HGVS string as an :class:`HgvsField`, attaching a parsed block when it is a placeable + simple substitution. ``None`` for an absent string.""" + if not hgvs: + return None + + block = parse_simple_substitution(hgvs) + if block is None: + return HgvsField(hgvs=hgvs) + + return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) + + +def mapped_triple(*, hgvs_c: Optional[str], hgvs_g: Optional[str], hgvs_p: Optional[str]) -> MappedTriple: + """Wrap a measured projection group's HGVS strings as a :class:`MappedTriple`. + + The group describes the measured variant at every level it has, the measured level included: a + protein assay's group holds only ``hgvs_p``, and a nucleotide assay's holds its normalized measured + string and the projections. A record whose group is not yet written (between mapping and the + projection step) gets an empty triple. + """ + return MappedTriple( + genomic=_hgvs_field_for_str(hgvs_g), + cdna=_hgvs_field_for_str(hgvs_c), + protein=_hgvs_field_for_str(hgvs_p), + ) + + +def get_lean_score_set_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> list[LeanVariantRecord]: + """Assemble the lean whole-set view for ``score_set``, one record per variant, ordered by variant + number. Unmapped variants are retained (null mapped fields) so the table and score histogram still + see them. ``as_of`` defaults to currently-live rows. + + Every per-variant lookup is a ``LATERAL ... LIMIT 1`` subquery. As plain joins, Postgres may choose a + hash join that scans the whole of ``mapping_record_alleles`` once its row estimate for the score set + crosses a threshold, which is orders of magnitude slower when that table is not cached. A lateral subquery with a + ``LIMIT`` cannot be flattened into the join tree, so each lookup stays a per-variant index probe + whatever the planner's statistics say. Each lookup yields at most one row by invariant (one live + record per variant, one live authoritative link per record, which points at one group); the ``ORDER BY`` + only makes a violated invariant deterministic. + """ + # The variant's live mapping record. + record = ( + select(MappingRecord.id, MappingRecord.assay_level) + .where(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of)) + .order_by(MappingRecord.id) + .limit(1) + .lateral("record") + ) + + # The authoritative (measured) allele, with its VEP consequence and its measured projection group's + # HGVS. ``is_authoritative`` is tested as a bare boolean: ``IS TRUE`` would not match the partial unique + # index's ``WHERE is_authoritative``. + authoritative_link = aliased(MappingRecordAllele) + authoritative = ( + select( + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + ProjectionGroup.hgvs_c, + ProjectionGroup.hgvs_g, + ProjectionGroup.hgvs_p, + ) + .select_from(authoritative_link) + .join(Allele, Allele.id == authoritative_link.allele_id) + .outerjoin( + VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) + ) + .outerjoin( + ProjectionGroup, + and_(ProjectionGroup.id == authoritative_link.projection_group, ProjectionGroup.live_at(as_of)), + ) + .where( + authoritative_link.mapping_record_id == record.c.id, + authoritative_link.is_authoritative, + authoritative_link.live_at(as_of), + ) + .order_by(authoritative_link.id) + .limit(1) + .lateral("authoritative") + ) + + statement = ( + select( + Variant.urn, + Variant.data, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + record.c.assay_level, + authoritative.c.vrs_digest, + authoritative.c.clingen_allele_id, + authoritative.c.functional_consequence, + authoritative.c.hgvs_c, + authoritative.c.hgvs_g, + authoritative.c.hgvs_p, + ) + .select_from(Variant) + # LEFT joins so unmapped variants stay (null mapped fields) for the table + score histogram. + .outerjoin(record, true()) + .outerjoin(authoritative, true()) + .where(Variant.score_set_id == score_set.id) + # Natural table order: variant number (the integer after '#'), id breaks ties stably. + .order_by(Variant.variant_number, Variant.id) + ) + + return [ + LeanVariantRecord( + variant_urn=row.urn, + score=score_from_variant_data(row.data), + consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + assay_level_digest=row.vrs_digest, + hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), + hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), + hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), + assay_level=_level(row.assay_level), + mapped=mapped_triple(hgvs_c=row.hgvs_c, hgvs_g=row.hgvs_g, hgvs_p=row.hgvs_p), + ) + for row in db.execute(statement) + ] diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 8a3624543..64b6dd034 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -1,7 +1,8 @@ import logging from collections import Counter, defaultdict +from datetime import datetime from operator import attrgetter -from typing import TYPE_CHECKING, BinaryIO, Optional, Sequence +from typing import TYPE_CHECKING, BinaryIO, Iterable, Optional, Sequence import numpy as np import pandas as pd @@ -23,6 +24,7 @@ from mavedb.lib.permissions import Action, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list +from mavedb.models.allele import Allele from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -32,7 +34,8 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier from mavedb.models.refseq_offset import RefseqOffset @@ -487,6 +490,19 @@ def find_meta_analyses_for_experiment_sets(db: Session, urns: list[str]) -> list ) +def readable_score_set_urns(user_data: Optional[UserData], score_sets: Iterable[ScoreSet]) -> set[str]: + """Return the URNs of ``score_sets`` the caller may read. + + For narrowing URN lists that view models build from ORM relationships with no knowledge of the caller, + such as ``meta_analyzed_by_score_set_urns`` or a collection's ``score_set_urns``. + """ + return { + score_set.urn + for score_set in score_sets + if score_set.urn is not None and has_permission(user_data, score_set, Action.READ).permitted + } + + def find_superseded_score_set_tail( score_set: ScoreSet, action: Optional["Action"] = None, user_data: Optional["UserData"] = None ) -> Optional[ScoreSet]: @@ -544,35 +560,50 @@ def find_publish_or_private_superseded_score_set_tail( return score_set -def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) -> Sequence[MappedVariant]: - """ - Load the current mapped variants for a score set with the relationships required to build VA-Spec - annotations eagerly loaded. - - This is the single source of truth for the eager-load shape shared by the annotated-variant - streaming endpoints and the public data export. The annotation builders reach through - ``MappedVariant.variant.score_set`` for publications, contributors, license, experiment, and score - calibrations, so each of those is loaded up front to avoid per-variant lazy loads. +def get_annotatable_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> Sequence[Variant]: + """Load a score set's annotatable variants from its variant's allele-graph, with the score-set + relationships the VA-Spec builders need eagerly loaded. + + A variant is *annotatable* when it has a live ``MappingRecord`` whose authoritative allele carries a + ``post_mapped`` VRS — exactly the condition under which :func:`annotation.context.variant_annotation_context` + yields a context. This is the single source of truth for the eager-load shape shared by the + annotated-variant streaming endpoints and the public data export. The annotation builders reach through + ``variant.score_set`` for publications, contributors, license, experiment, and score calibrations, so + each of those is loaded up front to avoid per-variant lazy loads. + + ``as_of`` reconstructs the annotatable set as it stood at a past instant: the same half-open + ``[valid_from, valid_to)`` predicate is applied to both the record and the authoritative link, so the + set is evaluated at one moment (matching :func:`annotation.context.variant_annotation_context`'s own + ``as_of``, so a per-variant context can be rebuilt at the same instant downstream). Defaults to the + currently-live rows. """ return ( - db.query(MappedVariant) - .join(MappedVariant.variant) - .join(Variant.score_set) - .filter(Variant.score_set_id == score_set.id) - .filter(MappedVariant.current.is_(True)) - .options( - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.publication_identifier_associations), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.created_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.modified_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.license), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.experiment), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.score_calibrations), + db.scalars( + select(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, and_(Allele.id == MappingRecordAllele.allele_id, Allele.post_mapped.isnot(None))) + .where(Variant.score_set_id == score_set.id) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.publication_identifier_associations), + selectinload(Variant.score_set).selectinload(ScoreSet.created_by), + selectinload(Variant.score_set).selectinload(ScoreSet.modified_by), + selectinload(Variant.score_set).selectinload(ScoreSet.license), + selectinload(Variant.score_set).selectinload(ScoreSet.experiment), + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + ) + .distinct() ) + .unique() .all() ) @@ -782,8 +813,6 @@ def refresh_variant_urns(db: Session, score_set: ScoreSet): variant.urn = refreshed_urn db.add(variant) - db.commit() - def bulk_create_urns(n, score_set, reset_counter=False) -> list[str]: start_value = 0 if reset_counter else score_set.num_variants diff --git a/src/mavedb/lib/seqrepo.py b/src/mavedb/lib/seqrepo.py index 708d08e2a..4330a9d85 100644 --- a/src/mavedb/lib/seqrepo.py +++ b/src/mavedb/lib/seqrepo.py @@ -11,7 +11,7 @@ # TODO (https://github.com/VariantEffect/mavedb-api/issues/354). We need pydantic upgraded to use this package. # from ga4gh.core.identifiers import is_ga4gh_identifier, CURIE_NAMESPACE as ga4gh_namespace -from typing import Generator, Optional, Union +from typing import Generator, Optional from biocommons.seqrepo import SeqRepo, __version__ as seqrepo_dep_version from bioutils.accessions import infer_namespaces @@ -20,6 +20,14 @@ DEFAULT_CHUNK_SIZE = 8192 +class SequenceNotFoundError(LookupError): + """Raised when SeqRepo holds no sequence for an accession.""" + + +class AmbiguousSequenceError(LookupError): + """Raised when SeqRepo holds more than one sequence for an accession.""" + + def base64url_to_hex(s: str) -> str: return hexlify(urlsafe_b64decode(s)).decode("ascii") @@ -33,13 +41,15 @@ def get_sequence_ids(sr: SeqRepo, query: str) -> list[str]: The query may be: * A fully-qualified sequence alias (e.g., VMC:0123 or refseq:NM_01234.5) - * A digest or digest prefix from VMC, TRUNC512, or MD5 + * A complete MD5 or TRUNC512 hex digest * A sequence accession (without namespace) - The first match will be returned. + The first match will be returned. Matching is exact: the shared SeqRepo holds the target sequences of + unpublished score sets, so neither digest prefixes nor SQL wildcards may be used to enumerate it. """ nsa_options = _generate_nsa_options(query) + aliases: list[dict] = [] for ns, a in nsa_options: aliases = list(sr.aliases.find_aliases(namespace=ns, alias=a)) if aliases: @@ -49,7 +59,30 @@ def get_sequence_ids(sr: SeqRepo, query: str) -> list[str]: return seq_ids -def _generate_nsa_options(query: str) -> Union[list[tuple[str, ...]], list[tuple[None, str]]]: +def resolve_refget(sr: SeqRepo, accession: str) -> str: + """Return the single GA4GH refget (``SQ.…``) SeqRepo holds for an accession. + + SeqRepo's ``seq_id`` is the sequence's sha512t24u digest, so the refget is ``SQ.`` plus it. The + shared SeqRepo is the only authority for what an accession's sequence is, and every writer of VRS + alleles must agree with it for allele digests to deduplicate. More than one sequence for an accession + is an error rather than a pick. + + The same contract is held by ``dcd_mapping.lookup.resolve_refget``; both are pinned to + ``tests/fixtures/canonical_accession_sequences.json``. + + :raise SequenceNotFoundError: if SeqRepo has no sequence for the accession + :raise AmbiguousSequenceError: if it has more than one + """ + seq_ids = get_sequence_ids(sr, accession) + if not seq_ids: + raise SequenceNotFoundError(f"SeqRepo has no sequence for accession {accession}.") + if len(seq_ids) > 1: + raise AmbiguousSequenceError(f"SeqRepo holds {len(seq_ids)} sequences for accession {accession}.") + + return f"SQ.{seq_ids[0]}" + + +def _generate_nsa_options(query: str) -> list[tuple[Optional[str], str]]: """ >>> _generate_nsa_options("NM_000551.3") [('refseq', 'NM_000551.3')] @@ -63,20 +96,25 @@ def _generate_nsa_options(query: str) -> Union[list[tuple[str, ...]], list[tuple >> _generate_nsa_options("SQ.test") [('ga4gh', 'test')] - >>> _generate_nsa_options("01234abcde") - [('MD5', '01234abcde%'), ('VMC', 'GS_ASNKvN4=%')] + >>> _generate_nsa_options("0123456789abcdef0123456789abcdef") + [('MD5', '0123456789abcdef0123456789abcdef')] + + >>> _generate_nsa_options("NM_%") + [] """ + # SeqRepo switches to a LIKE comparison whenever an argument contains "%". + if "%" in query: + return [] if ":" in query: # interpret as fully-qualified identifier - nsa_options = [tuple(query.split(sep=":", maxsplit=1))] - return nsa_options + namespace, alias = query.split(sep=":", maxsplit=1) + return [(namespace, alias)] namespaces = infer_namespaces(query) if namespaces: - nsa_options = [(ns, query) for ns in namespaces] - return nsa_options + return [(ns, query) for ns in namespaces] # TODO (https://github.com/VariantEffect/mavedb-api/issues/354). We need pydantic upgraded to use this package. # if ga4gh, try ga4gh. GA4GH only accepts identifiers with a namespace prefix, @@ -85,13 +123,12 @@ def _generate_nsa_options(query: str) -> Union[list[tuple[str, ...]], list[tuple # nsa_options = [("ga4gh", query)] # return nsa_options - # if hex, try MD5 - if re.match(r"^(?:[0-9A-Fa-f]{8,})$", query): - nsa_options = [("MD5", query + "%")] - # TRUNC512 isn't in seqrepo; synthesize equivalent VMC - id_b64u = hex_to_base64url(query) - nsa_options += [("VMC", "GS_" + id_b64u + "%")] - return nsa_options + # A complete MD5 (32 hex characters) or TRUNC512 (48) digest. TRUNC512 isn't in SeqRepo; synthesize the + # equivalent VMC alias. + if re.fullmatch(r"[0-9A-Fa-f]{32}", query): + return [("MD5", query.lower())] + if re.fullmatch(r"[0-9A-Fa-f]{48}", query): + return [("VMC", "GS_" + hex_to_base64url(query))] return [(None, query)] diff --git a/src/mavedb/lib/term_systems.py b/src/mavedb/lib/term_systems.py new file mode 100644 index 000000000..a5f826f53 --- /dev/null +++ b/src/mavedb/lib/term_systems.py @@ -0,0 +1,38 @@ +"""Shared building block for GA4GH ``Coding`` objects drawn from a named term system. + +A :data:`TermSystem` pairs a system URI/CURIE with the short prefix used to build a ``Coding.id`` for +codes drawn from it, kept together so a term's id and system can't drift apart the way two independent +lookup tables could. +""" + +from ga4gh.core.models import Coding, iriReference + +TermSystem = tuple[str, str] + + +### MaveDB Term Systems + +# MaveDB's own Cat-VRS relation codes (member -> defining), in cat_vrs.py's own framing rather than the +# spec's. +MAVEDB_CAT_VRS_RELATION: TermSystem = ("https://mavedb.org/cat-vrs/relations", "mavedb") + + +### External Term Systems + +# External variant registries. Identifiers consumers match on, so deliberately not built from the registries' +# link constants: a registry moving its web pages must not rename a system in served output. +CLINVAR_VARIATION: TermSystem = ("https://www.ncbi.nlm.nih.gov/clinvar/variation/", "clinvar") +CLINGEN_ALLELE_REGISTRY: TermSystem = ("https://reg.clinicalgenome.org/", "clingen") +GNOMAD: TermSystem = ("https://gnomad.broadinstitute.org", "gnomad") + + +def coding(term_system: TermSystem, code: str, *, iri: str | None = None, system_version: str | None = None) -> Coding: + """Build a ``Coding`` for `code`, drawn from `term_system` at `system_version`, optionally resolvable at `iri`.""" + uri, prefix = term_system + return Coding( + id=f"{prefix}:{code}", + code=code, + system=uri, + systemVersion=system_version, + iris=[iriReference(root=iri)] if iri is not None else None, + ) diff --git a/src/mavedb/lib/types/score_calibrations.py b/src/mavedb/lib/types/score_calibrations.py index d40edaf27..06212e825 100644 --- a/src/mavedb/lib/types/score_calibrations.py +++ b/src/mavedb/lib/types/score_calibrations.py @@ -1,6 +1,81 @@ -from typing import TypedDict +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date +from typing import Any, Optional, TypedDict + +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus class ClassificationDict(TypedDict): indexed_by: str classifications: dict[str, set[str]] + + +VariantIdentity = tuple[Optional[str], Optional[str], Optional[str]] +"""A variant's ``(hgvs_nt, hgvs_pro, hgvs_splice)`` tuple. + +Variant URNs are positional within a score set (``{score_set.urn}#{n}``, renumbered from one on every +upload), so ``#5`` can name a different biological variant after a re-upload. The HGVS tuple is the +identity that survives one, and so is what calibration variant references are re-resolved against. +""" + + +@dataclass(frozen=True) +class CalibrationControlSnapshot: + """One calibration control, recorded by variant identity instead of variant id. + + Audit provenance is carried alongside the identity so a relinked control keeps the authorship and + date of the original submission rather than appearing to have been entered by whoever re-uploaded + the scores. + """ + + identity: VariantIdentity + clinical_status: CalibrationControlStatus + created_by_id: int + creation_date: date + + +@dataclass +class CalibrationVariantLinkSnapshot: + """The variant references held by one calibration that a re-upload cannot reconstruct. + + Only irreproducible references are captured. Range-based bin membership is omitted on purpose: it + is a function of the variants' scores, so it is recomputed from the new upload rather than + remembered (see :func:`~mavedb.lib.score_calibrations.restore_calibration_variant_links`). + """ + + calibration_id: int + controls: list[CalibrationControlSnapshot] = field(default_factory=list) + + # Class-based bin membership keyed by functional classification id. Classification rows survive a + # re-upload, so their ids remain valid addresses when the membership is restored. + classification_members: dict[int, list[VariantIdentity]] = field(default_factory=dict) + + +@dataclass +class CalibrationVariantRelinkReport: + """Outcome of re-establishing a score set's calibration variant references after a re-upload. + + Relinked and dropped counts describe references carried across by identity (controls and + class-based bin membership). Re-binned counts describe range-based membership recomputed from the + new scores, where "dropped" has no meaning — the bin simply holds whatever now falls in it. + """ + + controls_relinked: int = 0 + controls_dropped: int = 0 + classification_members_relinked: int = 0 + classification_members_dropped: int = 0 + classifications_rebinned: int = 0 + classification_members_rebinned: int = 0 + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serializable summary suitable for a job's logging context.""" + return { + "calibration_controls_relinked": self.controls_relinked, + "calibration_controls_dropped": self.controls_dropped, + "calibration_classification_members_relinked": self.classification_members_relinked, + "calibration_classification_members_dropped": self.classification_members_dropped, + "calibration_classifications_rebinned": self.classifications_rebinned, + "calibration_classification_members_rebinned": self.classification_members_rebinned, + } diff --git a/src/mavedb/lib/types/workflow.py b/src/mavedb/lib/types/workflow.py index 459d4337d..215adbf14 100644 --- a/src/mavedb/lib/types/workflow.py +++ b/src/mavedb/lib/types/workflow.py @@ -75,6 +75,7 @@ class JobDefinition(TypedDict): type: str function: str params: dict[str, Any] + optional_params: NotRequired[dict[str, Any]] dependencies: list[tuple[str, DependencyType]] retry_delay_seconds: NotRequired[int] diff --git a/src/mavedb/lib/uniprot/utils.py b/src/mavedb/lib/uniprot/utils.py index 7f87524b2..15c96faf3 100644 --- a/src/mavedb/lib/uniprot/utils.py +++ b/src/mavedb/lib/uniprot/utils.py @@ -1,11 +1,11 @@ from typing import Literal, Union -from mavedb.lib.validation.identifier import validate_refseq_identifier +from mavedb.lib.validation.identifier import validate_ensembl_identifier, validate_refseq_identifier def infer_db_name_from_sequence_accession( sequence_accession: str, -) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein"]]: +) -> Union[Literal["RefSeq_Nucleotide", "RefSeq_Protein", "Ensembl_Protein", "Ensembl_Transcript"]]: """ Infers the database name from a sequence accession. @@ -15,11 +15,41 @@ def infer_db_name_from_sequence_accession( Returns: str: The inferred database name. """ - validate_refseq_identifier(sequence_accession) - - if sequence_accession.startswith("NM_"): + if sequence_accession.startswith("ENSP"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Protein" + elif sequence_accession.startswith("ENST"): + validate_ensembl_identifier(sequence_accession) + return "Ensembl_Transcript" + elif sequence_accession.startswith("NM_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Nucleotide" - if sequence_accession.startswith("NP_"): + elif sequence_accession.startswith("NP_"): + validate_refseq_identifier(sequence_accession) return "RefSeq_Protein" - raise NotImplementedError("Only RefSeq NM and NP identifiers are currently supported for inference.") + raise NotImplementedError( + "Only RefSeq (NM_/NP_) and Ensembl (ENSP/ENST) identifiers are currently supported for inference." + ) + + +def id_mapping_query_accession(sequence_accession: str, from_db: str) -> str: + """ + Returns the form of a sequence accession to submit to UniProt ID mapping. + + UniProt's RefSeq_Nucleotide index matches versioned NM_ accessions exactly and lags RefSeq + releases (e.g. it holds NM_002878.3 while RefSeq and the UniProtKB entry are at NM_002878.4), + so current transcript versions often return no results. Unversioned NM_ accessions match + reliably. The other supported databases match versioned accessions, so those pass through. + + Args: + sequence_accession (str): The sequence accession to submit. + from_db (str): The UniProt ID mapping database the accession belongs to. + + Returns: + str: The accession to submit. + """ + if from_db == "RefSeq_Nucleotide": + return sequence_accession.split(".", 1)[0] + + return sequence_accession diff --git a/src/mavedb/lib/urn_redirects.py b/src/mavedb/lib/urn_redirects.py new file mode 100644 index 000000000..d131d38ab --- /dev/null +++ b/src/mavedb/lib/urn_redirects.py @@ -0,0 +1,172 @@ +"""Forwarding of URNs that publication has retired. + +A dataset is created with a ``tmp:`` URN and keeps it until it is published, at which point +:func:`mavedb.routers.score_sets.publish_score_set` overwrites the URN in place with a permanent one. +Any link already shared to the unpublished record then stops resolving, which is what +https://github.com/VariantEffect/mavedb-ui/issues/617 reports: the record is still there, under a name +the caller has no way to guess. + +Publication records what each retired URN became, and every read is checked here before it reaches its +route. A request naming a retired URN is answered with ``308 Permanent Redirect`` to the same path +under the record's current URN. + +Three limits: + + - Only a public target is forwarded to. A ``Location`` header names the record it points at, and + forwarding happens before any route checks a permission, so the header would disclose that URN to + an anonymous caller. See _target_is_public. + - Forwarding is one hop. Nothing in the application renames a published record, so a chain cannot + arise; if one ever could, resolution here would need to follow it. + - URNs retired before this was added are unrecoverable. Publication overwrote them and no history of + them was kept, so links to records published earlier stay broken. +""" + +import logging +from typing import Optional +from urllib.parse import quote + +from fastapi import Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session +from starlette.requests import Request + +from mavedb.deps import get_db +from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.actions import Action +from mavedb.lib.permissions.core import has_permission +from mavedb.lib.validation.urn_re import ( + MAVEDB_EXPERIMENT_SET_URN_RE, + MAVEDB_EXPERIMENT_URN_RE, + MAVEDB_SCORE_SET_URN_RE, + MAVEDB_TMP_URN_RE, +) +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.score_set import ScoreSet +from mavedb.models.urn_redirect import UrnRedirect + +logger = logging.getLogger(__name__) + +# Methods whose requests are forwarded. See forward_retired_urns for why a write is not. +SAFE_METHODS = frozenset({"GET", "HEAD"}) + + +def record_urn_redirect(db: Session, old_urn: Optional[str], new_urn: str) -> None: + """ + Record that a record's URN has changed, so that requests naming the old one can be forwarded. + + Staged on the session rather than committed, so that a caller which reassigns several URNs -- as + publication does, across an experiment set, an experiment and a score set -- commits the redirects + together with the renames they describe. + + :param db: An active database session. + :param old_urn: The URN being retired. A record that never had one, or a rename that is not a + change, is not worth a row and is ignored. + :param new_urn: The URN replacing it. + """ + if not old_urn or old_urn == new_urn: + return + + db.add(UrnRedirect(old_urn=old_urn, new_urn=new_urn)) # type: ignore[call-arg] + + +def _target_is_public(db: Session, urn: str) -> bool: + """ + Report whether the record a redirect points to is one that may be named to any caller. + + A ``Location`` header discloses the URN it carries, to whoever asked -- including an anonymous + caller, since forwarding happens before a route checks anything. Publication only ever records a + redirect onto a record it is making public, and nothing in the application returns a published + record to private, so a private target should not arise; a row written out of band, or by some + later feature, would be enough for one to. Deferring to ``Action.READ`` with no user keeps that + from becoming a disclosure, and keeps this answer in step with the rules a route would apply. + + A target that no longer exists is likewise not public: a deleted record leaves its redirect row + behind, and forwarding to it would answer a permanent redirect with a 404. + + :param db: An active database session. + :param urn: The URN a redirect points to. + :return: True only if a record under this URN exists and an anonymous caller may read it. + """ + target: Optional[ScoreSet | Experiment | ExperimentSet] = None + if MAVEDB_SCORE_SET_URN_RE.fullmatch(urn): + target = db.query(ScoreSet).filter(ScoreSet.urn == urn).one_or_none() + elif MAVEDB_EXPERIMENT_URN_RE.fullmatch(urn): + target = db.query(Experiment).filter(Experiment.urn == urn).one_or_none() + elif MAVEDB_EXPERIMENT_SET_URN_RE.fullmatch(urn): + target = db.query(ExperimentSet).filter(ExperimentSet.urn == urn).one_or_none() + + return target is not None and has_permission(None, target, Action.READ).permitted + + +def forwarded_path(db: Session, path: str) -> Optional[str]: + """ + Rewrite a request path so that any retired URN in it names the record's current URN instead. + + Substitution is by substring, not by path segment, so a variant URN -- ``{score_set_urn}#{n}`` -- + is carried along by its score set's redirect. + + :param db: An active database session. + :param path: The decoded request path. + :return: The rewritten path, or None if the path should be served as it is: it names no retired URN, + or one whose target this caller must not be told about. See _target_is_public. + """ + # A live temporary URN belongs to an unpublished record and matches nothing in the table, so the + # lookup below distinguishes the two cases and no separate check for publication is needed. + candidate_urns = set(MAVEDB_TMP_URN_RE.findall(path)) + if not candidate_urns: + return None + + redirects = db.execute( + select(UrnRedirect.old_urn, UrnRedirect.new_urn).where(UrnRedirect.old_urn.in_(candidate_urns)) + ).all() + if not redirects: + return None + + forwarded = path + for old_urn, new_urn in redirects: + if not _target_is_public(db, new_urn): + return None + forwarded = forwarded.replace(old_urn, new_urn) + + return forwarded + + +def forward_retired_urns(request: Request, db: Session = Depends(get_db)) -> None: + """ + Forward a request that names a retired URN to the same resource under its current URN. + + Installed as an application-wide dependency in :mod:`mavedb.server_main`, which is what makes one + implementation cover every route that takes a URN, sub-resources included: a stale link to a score + set's scores CSV or mapped variants is forwarded on the same terms as a link to the score set. + + Only reads are forwarded. A write to a retired URN keeps getting the 404 it gets today, which + tells the client to look the record up again. + """ + if request.scope["method"] not in SAFE_METHODS: + return + + # The ASGI scope rather than request.url: Starlette builds request.url by re-parsing the decoded + # path, so a variant URN's '#' starts a fragment there and everything after it -- the variant + # number, the sub-resource, the query -- is silently dropped. + path = request.scope["path"] + query = request.scope.get("query_string", b"").decode("ascii") + + forwarded = forwarded_path(db, path) + if forwarded is None: + return + + # The scope's path is percent-decoded, so a '#' in it has to be re-encoded or it would open a + # fragment in the header. Relative, so that a proxy's scheme and host survive. + location = quote(forwarded, safe="/:") + if query: + location = f"{location}?{query}" + + save_to_logging_context({"requested_resource": path, "forwarded_to": forwarded}) + logger.info(msg="Forwarding a request that named a retired URN.", extra=logging_context()) + + raise HTTPException( + status_code=308, + detail="This URN was replaced when the record was published; the record has moved permanently.", + headers={"Location": location}, + ) diff --git a/src/mavedb/lib/urns.py b/src/mavedb/lib/urns.py index 46be37a10..e67ad26d1 100644 --- a/src/mavedb/lib/urns.py +++ b/src/mavedb/lib/urns.py @@ -4,7 +4,7 @@ from typing import Optional from uuid import uuid4 -from sqlalchemy import func +from sqlalchemy import func, text from sqlalchemy.orm import Session from mavedb.lib.validation.urn_re import ( @@ -18,13 +18,26 @@ logger = logging.getLogger(__name__) +def lock_urn_assignment(db: Session) -> None: + """ + Serialize permanent URN assignment for the rest of the current transaction. + + The ``generate_*_urn`` functions read the highest assigned URN and add one, so under READ COMMITTED two concurrent + publications would mint the same URN. Take this before loading the records being published; the lock is released + when the transaction commits or rolls back. + + :param db: An active database session + """ + db.execute(text("SELECT pg_advisory_xact_lock(hashtext('mavedb.urn_assignment'))")) + + def generate_experiment_set_urn(db: Session): """ Generate a new URN for an experiment set. Experiment set URNs include an 8-digit, zero-padded, sequentially-assigned numeric part. This function finds the - maximum value in the database and adds one to form the new URN. To ensure atomicity, it should be called in the - context of a database transaction. + maximum value in the database and adds one to form the new URN. Callers must hold ``lock_urn_assignment`` so that + concurrent publications cannot read the same maximum. :param db: An active database session :return: The next available experiment set URN @@ -54,8 +67,8 @@ def generate_experiment_urn(db: Session, experiment_set: ExperimentSet, experime a, b, ..., z, aa, ab, ..., az, ba, ... bz, ... zz, aaa, ..., zzz, aaaa, ... ``` This function looks at the database records for other experiments in the set and finds the maximum value of the - alphabetic part, then increments it to form the new URN. To ensure atomicity, it should be called in the context of - a database transaction to ensure atomicity. + alphabetic part, then increments it to form the new URN. Callers must hold ``lock_urn_assignment`` so that + concurrent publications cannot read the same maximum. For meta-analyses, the suffix is always 0. There can only be one meta-analysis per experiment set. @@ -110,8 +123,8 @@ def generate_score_set_urn(db: Session, experiment: Experiment): zero-padded to a fixed width. This function looks at the database records for other scoresets belonging to the experiment and finds the maximum - value of the numeric part, then increments it to form the new URN. To ensure atomicity, it should be called in the - context of a database transaction. + value of the numeric part, then increments it to form the new URN. Callers must hold ``lock_urn_assignment`` so that + concurrent publications cannot read the same maximum. :param db: An active database session :param experiment: The experiment to which this score set belongs diff --git a/src/mavedb/lib/validation/constants/general.py b/src/mavedb/lib/validation/constants/general.py index 22ca4cbf5..14a087fc7 100644 --- a/src/mavedb/lib/validation/constants/general.py +++ b/src/mavedb/lib/validation/constants/general.py @@ -46,6 +46,7 @@ calibration_variant_column_name = "variant_urn" calibration_class_column_name = "class_name" +calibration_control_status_column_name = "clinical_status" valid_dataset_columns = [score_columns, count_columns] valid_variant_columns = [variant_score_data, variant_count_data] diff --git a/src/mavedb/lib/validation/dataframe/calibration.py b/src/mavedb/lib/validation/dataframe/calibration.py index 1c46be46b..7594c36c5 100644 --- a/src/mavedb/lib/validation/dataframe/calibration.py +++ b/src/mavedb/lib/validation/dataframe/calibration.py @@ -4,6 +4,7 @@ from mavedb.lib.validation.constants.general import ( calibration_class_column_name, + calibration_control_status_column_name, calibration_variant_column_name, hgvs_nt_column, hgvs_pro_column, @@ -11,6 +12,7 @@ from mavedb.lib.validation.dataframe.column import validate_data_column, validate_variant_column from mavedb.lib.validation.dataframe.dataframe import standardize_dataframe, validate_no_null_rows from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.view_models import score_calibration @@ -22,6 +24,13 @@ hgvs_pro_column, ) +STANDARD_CALIBRATION_CONTROL_COLUMNS = ( + calibration_variant_column_name, + calibration_control_status_column_name, + hgvs_nt_column, + hgvs_pro_column, +) + def validate_and_standardize_calibration_classes_dataframe( db: Session, @@ -246,3 +255,140 @@ def validate_calibration_classes( unprovided_classes = defined_classes - provided_classes if unprovided_classes: raise ValidationError("Some defined classes in the calibration are missing from the classes file.") + + +def validate_and_standardize_calibration_controls_dataframe( + db: Session, + score_set: ScoreSet, + controls_df: pd.DataFrame, +) -> list[score_calibration.CalibrationControlCreate]: + """Validate a calibration controls CSV and convert it to ``CalibrationControlCreate`` rows. + + Mirrors :func:`validate_and_standardize_calibration_classes_dataframe`. The CSV must carry a + variant identifier column (``variant_urn``, ``hgvs_nt``, or ``hgvs_pro``) and a + ``clinical_status`` column. Every variant must belong to the score set, and every status must be + a ``CalibrationControlStatus`` value (case-insensitive). The chosen variant column is resolved to + MaveDB URNs so the returned controls carry ``variant_urn``, matching the inline-JSON contract. + + Args: + db: Database session for validation and URN resolution. + score_set: The score set the calibration belongs to. + controls_df: The parsed controls CSV. + + Returns: + One ``CalibrationControlCreate`` per row. + + Raises: + ValidationError: On unexpected/missing columns, invalid variants, or invalid status values. + """ + standardized_df = standardize_dataframe(controls_df, STANDARD_CALIBRATION_CONTROL_COLUMNS) + validate_calibration_control_df_column_names(standardized_df) + validate_no_null_rows(standardized_df) + + column_mapping = {c.lower(): c for c in standardized_df.columns} + index_column = choose_calibration_index_column(standardized_df) + + # Drop rows without a clinical status. + standardized_df = standardized_df.dropna( + subset=[column_mapping[calibration_control_status_column_name]] + ).reset_index(drop=True) + + for c in column_mapping: + if c in {calibration_variant_column_name, hgvs_nt_column, hgvs_pro_column}: + validate_variant_column(standardized_df[c], column_mapping[c] == index_column) + elif c == calibration_control_status_column_name: + validate_data_column(standardized_df[c], force_numeric=False) + validate_calibration_control_statuses(standardized_df[column_mapping[c]]) + + if column_mapping[c] == index_column: + validate_index_existence_in_score_set(db, score_set, standardized_df[column_mapping[c]], column_mapping[c]) + + return resolve_calibration_controls(db, score_set, standardized_df, index_column) + + +def validate_calibration_control_df_column_names(df: pd.DataFrame) -> None: + """Validate the column names of a calibration controls DataFrame. + + Requires a ``clinical_status`` column and at least one variant identifier column, and rejects + unexpected columns. Comparison is case-insensitive. Mirrors + :func:`validate_calibration_df_column_names`. + """ + if any(type(c) is not str for c in df.columns): + raise ValidationError("column names must be strings") + + if any(c.isspace() for c in df.columns) or any(len(c) == 0 for c in df.columns): + raise ValidationError("column names cannot be empty or whitespace") + + if len(df.columns) != len(set(c.lower() for c in df.columns)): + raise ValidationError("duplicate column names are not allowed (case-insensitive)") + + columns = [c.lower() for c in df.columns] + + if calibration_control_status_column_name not in columns: + raise ValidationError(f"missing required column: '{calibration_control_status_column_name}'") + + if set(columns).isdisjoint({hgvs_nt_column, hgvs_pro_column, calibration_variant_column_name}): + raise ValidationError( + f"at least one of {', '.join({hgvs_nt_column, hgvs_pro_column, calibration_variant_column_name})} must be present" + ) + + +def validate_calibration_control_statuses(statuses: pd.Series) -> None: + """Ensure every ``clinical_status`` value maps to a ``CalibrationControlStatus`` (case-insensitive).""" + valid_statuses = {status.value for status in CalibrationControlStatus} + invalid_statuses = { + str(status) for status in statuses.tolist() if str(status).strip().lower() not in valid_statuses + } + if invalid_statuses: + raise ValidationError( + f"Invalid clinical status value(s): {', '.join(sorted(invalid_statuses))}. " + f"Allowed values are: {', '.join(sorted(valid_statuses))}." + ) + + +def resolve_calibration_controls( + db: Session, + score_set: ScoreSet, + df: pd.DataFrame, + index_column: str, +) -> list[score_calibration.CalibrationControlCreate]: + """Convert a validated controls DataFrame to ``CalibrationControlCreate`` rows, resolving URNs. + + For a ``variant_urn`` index the URN is taken directly. For an HGVS index the values are mapped to + their variant URNs within the score set in a single query. A control is a single variant, so an + HGVS that matches more than one variant is ambiguous and rejected — whether a given HGVS column + collides depends on the score set's own index (``hgvs_pro`` is unique in a protein-indexed set but + not in a nucleotide-indexed one), so this is checked against the data rather than assumed. + """ + column_mapping = {c.lower(): c for c in df.columns} + status_column = column_mapping[calibration_control_status_column_name] + index_values = df[index_column].tolist() + + if index_column.lower() == calibration_variant_column_name: + urn_by_index_value = {value: value for value in index_values} + else: + hgvs_attribute = Variant.hgvs_nt if index_column.lower() == hgvs_nt_column else Variant.hgvs_pro + urns_by_hgvs: dict[str, list[str]] = {} + for urn, hgvs in db.execute( + select(Variant.urn, hgvs_attribute).where( + Variant.score_set_id == score_set.id, hgvs_attribute.in_(index_values) + ) + ).all(): + urns_by_hgvs.setdefault(hgvs, []).append(urn) + + ambiguous_identifiers = sorted(hgvs for hgvs, urns in urns_by_hgvs.items() if len(urns) > 1) + if ambiguous_identifiers: + raise ValidationError( + "These variant identifiers match more than one variant in the score set; " + f"use the variant URN instead: {', '.join(ambiguous_identifiers)}." + ) + + urn_by_index_value = {hgvs: urns[0] for hgvs, urns in urns_by_hgvs.items()} + + return [ + score_calibration.CalibrationControlCreate( + variant_urn=urn_by_index_value[row[index_column]], + clinical_status=CalibrationControlStatus(str(row[status_column]).strip().lower()), + ) + for _, row in df.iterrows() + ] diff --git a/src/mavedb/lib/validation/identifier.py b/src/mavedb/lib/validation/identifier.py index e6e2dcde7..077b13fc0 100644 --- a/src/mavedb/lib/validation/identifier.py +++ b/src/mavedb/lib/validation/identifier.py @@ -1,3 +1,5 @@ +import re + import idutils from mavedb.lib.validation.constants.identifier import valid_dbnames @@ -52,6 +54,9 @@ def validate_ensembl_identifier(identifier: str): """ Validates whether the identifier is a valid Ensembl identifier. + Ensembl stable IDs may carry a version suffix (e.g. ``ENSP00000369497.3``), which + ``idutils.is_ensembl`` does not accept; strip it before validating the stable ID. + Parameters __________ identifier : str @@ -62,7 +67,8 @@ def validate_ensembl_identifier(identifier: str): ValidationError If the identifier is not a valid Ensembl identifier. """ - if not idutils.is_ensembl(identifier): + base_identifier = re.sub(r"\.\d+$", "", identifier) + if not idutils.is_ensembl(base_identifier): raise ValidationError(f"'{identifier}' is not a valid Ensembl accession.") diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py new file mode 100644 index 000000000..22b6cd66e --- /dev/null +++ b/src/mavedb/lib/variant_detail.py @@ -0,0 +1,309 @@ +"""Assembles the variant-detail envelope backing ``GET /variants/{urn}``. + +The envelope has two tiers: flat, UI-ergonomic assay-level fields (HGVS pair, digest, ClinGen id), +and a spec-pure GA4GH ``CategoricalVariant`` (built by :mod:`lib.cat_vrs`) with a MaveDB layer +riding alongside it, keyed by VRS digest — per-allele identity (level, HGVS, ClinGen id, +member→defining relation) and external annotations (:mod:`lib.allele_annotations`). Also includes +the variant's functional ``classifications`` per calibration, and its version standing +(``is_current`` / ``superseded_by_score_set``), so a superseded variant self-describes. + +``as_of`` only reconstructs the **molecular** layer (Cat-VRS membership and VEP/gnomAD/ClinVar +annotations) at the past instant. Scores are immutable and calibrations carry no ``ValidTime``, so +both are always returned as they stand now, regardless of ``as_of``. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional, Sequence + +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleIdentity +from mavedb.lib.alleles import LiveRecordLinks, get_live_record_allele_links +from mavedb.lib.cat_vrs import build_categorical_variant, is_projection_partner, member_label +from mavedb.lib.score_calibrations import calibration_preference_key, classifications_by_variant +from mavedb.lib.score_set_variants import MappedTriple, mapped_triple +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class VariantClassificationRecord: + """One functional classification the variant belongs to, tagged with its calibration. + + ``calibration_id`` / ``primary`` locate it among the score set's calibrations (``primary`` is + the UI default). ``classification`` is the ORM + :class:`ScoreCalibrationFunctionalClassification`, serialized at the view-model boundary. + """ + + calibration_id: int + primary: bool + classification: ScoreCalibrationFunctionalClassification + + +@dataclass(frozen=True) +class VariantDetailInputs: + """The database inputs for a chunk of variants' detail envelopes, loaded up front. + + ``annotations`` is keyed by digest across the whole chunk; each envelope takes the entries for its own + links. :meth:`detail_for` assembles one variant's envelope without touching the database. + """ + + record_links: dict[int, LiveRecordLinks] + annotations: dict[str, AlleleAnnotations] + classifications: dict[int, list[VariantClassificationRecord]] + + def detail_for(self, variant: Variant, *, superseding_score_set: Optional[ScoreSet] = None) -> "VariantDetail": + """Assemble the variant-detail envelope for ``variant``. + + ``superseding_score_set`` is the newer version the caller already resolved for visibility + (``None`` if unreadable, mirroring ``fetch_score_set_by_urn``); its presence drives + ``is_current`` / ``superseded_by_score_set``. + """ + return _assemble_variant_detail( + variant, + self.record_links.get(variant.id), + self.annotations, + self.classifications.get(variant.id, []), + superseding_score_set=superseding_score_set, + ) + + +@dataclass(frozen=True) +class VariantDetail: + """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" + + urn: str + scores: Optional[dict[str, Any]] + counts: Optional[dict[str, Any]] + classifications: list[VariantClassificationRecord] + + # Flat, UI-ergonomic assay-level fields. + assay_level: Optional[SequenceLevel] + target_hgvs: Optional[str] # submitted, target/assay coordinates + reference_hgvs: Optional[str] # mapped, reference coordinates, assay level + assay_level_digest: Optional[str] + clingen_allele_id: Optional[str] + + # Raw GA4GH VRS pair, surfaced flat so a bulk/VRS consumer doesn't need to dig the measured + # allele out of the Cat-VRS below. + pre_mapped: Optional[dict[str, Any]] + post_mapped: Optional[dict[str, Any]] + + # Spec-pure GA4GH Cat-VRS (no MaveDB fields), plus the MaveDB layer alongside it: a per-allele + # identity sidecar keyed by VRS digest, one entry per linked allele (shares keys with + # `annotations`), carrying level, HGVS, ClinGen id, and relation. + molecular_representation: Optional[dict[str, Any]] + mode: Optional[str] # CatVrsMode value — projection | reverse_translation + # The measured variant's mapped HGVS at every level, from its measured projection group; includes + # levels with no allele in `alleles` (an intronic coding projection). + mapped: MappedTriple + alleles: dict[str, AlleleIdentity] + + # External annotations, keyed by VRS digest, joined to the Cat-VRS members / the alleles sidecar. + annotations: dict[str, AlleleAnnotations] + + # Version standing — self-descriptive for a superseded variant. + is_current: bool + # Score-set URN (not variant URN) of the version that supersedes this one, if any and readable. + # Supersession is versioned at the score-set level, and a newer version may add/drop/renumber + # variants, so consumers must look this variant up within that score set rather than follow a + # superseding-variant pointer. + superseded_by_score_set: Optional[str] + + +def _classifications_by_variant( + db: Session, variant_ids: Sequence[int], *, visible_calibration_ids: Optional[set[int]] +) -> dict[int, list[VariantClassificationRecord]]: + """Functional classifications each variant belongs to, one row per calibration that classifies it. + + Each variant's rows are ordered by ``calibration_preference_key`` (+ id), so the first entry matches the + UI's default. ``visible_calibration_ids`` restricts to readable calibrations (``None`` = no restriction, + for lib-level callers). + """ + return { + variant_id: [ + VariantClassificationRecord( + calibration_id=calibration.id, primary=calibration.primary, classification=classification + ) + for classification, calibration in sorted( + pairs, key=lambda pair: (*calibration_preference_key(pair[1]), pair[1].id) + ) + if visible_calibration_ids is None or calibration.id in visible_calibration_ids + ] + for variant_id, pairs in classifications_by_variant(db, variant_ids).items() + } + + +def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[SequenceLevel]) -> Optional[str]: + """Depositor-submitted HGVS in the variant's assay frame: ``hgvs_pro`` for a protein assay, + otherwise ``hgvs_nt`` (genomic and coding assays share this column; ``hgvs_splice`` is never an + index column).""" + if assay_level == SequenceLevel.protein.value: + return variant.hgvs_pro + return variant.hgvs_nt + + +def load_variant_detail_inputs( + db: Session, + variants: Sequence[Variant], + *, + visible_calibration_ids: Optional[set[int]] = None, + as_of: Optional[datetime] = None, +) -> VariantDetailInputs: + """Load the detail-envelope inputs for ``variants`` in six queries in total, however many variants there + are (where building each envelope alone would cost six per variant). + + Two for the records and links, three for the annotations of every linked allele, and one for the + classifications. Whole-set callers pass a chunk of variants (:data:`~mavedb.lib.alleles.VARIANT_CHUNK_SIZE`). + ``visible_calibration_ids`` restricts classifications to readable calibrations. ``as_of`` reconstructs + the molecular layer only — see the module docstring. + """ + variant_ids = [variant.id for variant in variants] + record_links = get_live_record_allele_links(db, variant_ids, as_of=as_of) + alleles = [link.allele for entry in record_links.values() for link in entry.links] + return VariantDetailInputs( + record_links=record_links, + annotations=get_allele_annotations(db, alleles, as_of=as_of), + classifications=_classifications_by_variant(db, variant_ids, visible_calibration_ids=visible_calibration_ids), + ) + + +def get_variant_detail( + db: Session, + variant: Variant, + *, + superseding_score_set: Optional[ScoreSet] = None, + visible_calibration_ids: Optional[set[int]] = None, + as_of: Optional[datetime] = None, +) -> VariantDetail: + """Assemble the variant-detail envelope for one variant: the single-variant form of + :func:`load_variant_detail_inputs` and :meth:`VariantDetailInputs.detail_for`.""" + inputs = load_variant_detail_inputs(db, [variant], visible_calibration_ids=visible_calibration_ids, as_of=as_of) + return inputs.detail_for(variant, superseding_score_set=superseding_score_set) + + +def _assemble_variant_detail( + variant: Variant, + record_links: Optional[LiveRecordLinks], + chunk_annotations: dict[str, AlleleAnnotations], + classifications: list[VariantClassificationRecord], + *, + superseding_score_set: Optional[ScoreSet], +) -> VariantDetail: + """Build one variant's envelope from its loaded record links, the chunk's annotations and its + classifications.""" + data = variant.data if isinstance(variant.data, dict) else {} + scores = data.get("score_data") + counts = data.get("count_data") + + # The live mapping record supplies the assay level and the mapped (reference-frame) assay HGVS. + record = record_links.record if record_links is not None else None + assay_level = SequenceLevel(record.assay_level) if record is not None else None + reference_hgvs = record.hgvs_assay_level if record is not None else None + + # The live allele links: the authoritative allele gives the assay-level digest + ClinGen id; all + # linked alleles seed the digest-keyed annotation map. + links = record_links.links if record_links is not None else [] + authoritative = next((link for link in links if link.is_authoritative), None) + measured_group = authoritative.group if authoritative is not None else None + assay_level_digest = authoritative.allele.vrs_digest if authoritative is not None else None + clingen_allele_id = authoritative.allele.clingen_allele_id if authoritative is not None else None + + annotations = { + link.allele.vrs_digest: chunk_annotations[link.allele.vrs_digest] + for link in links + if link.allele.vrs_digest in chunk_annotations + } + + # Spec-pure Cat-VRS built on the fly, plus its mode. + transit = build_categorical_variant( + links, + name=variant.urn or "", + cross_references={digest: entry.cross_references() for digest, entry in annotations.items()}, + ) + if transit is not None: + molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) + mode: Optional[str] = transit.mode.value + else: + molecular_representation = None + mode = None + + # Per-allele identity sidecar: one entry per linked allele, keyed by VRS digest (the same link + # set that seeds `annotations`, so the two maps share keys). Carries level, reference-frame HGVS + # (exactly one of hgvs_g/c/p is populated per allele), ClinGen id, and the member→defining relation. + + # projection_group -> the nucleotide digests sharing it, for within-record c↔g pairing. A nucleotide link's + # projection is the ≤1 *other* nucleotide digest in its group. The protein consequence in the measured group + # pairs with nothing: it is what the pair encodes (its relation says so), not another spelling of it. + group_digests: dict[int, list[str]] = {} + for link in links: + if ( + link.projection_group is not None + and link.allele.vrs_digest is not None + and link.allele.level in NUCLEOTIDE_LEVELS + ): + group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) + + # Members are labelled relative to the measured (authoritative) allele. + focus_level = authoritative.allele.level if authoritative is not None else None + focus_group = authoritative.projection_group if authoritative is not None else None + + alleles: dict[str, AlleleIdentity] = {} + for link in links: + allele = link.allele + + projection_of: Optional[str] = None + if link.projection_group is not None and allele.level in NUCLEOTIDE_LEVELS: + projection_of = next( + (digest for digest in group_digests.get(link.projection_group, []) if digest != allele.vrs_digest), + None, + ) + + relation, derivation = ( + (None, None) + if link.is_authoritative + else member_label( + focus_level, + allele.level, + is_projection=is_projection_partner(link.projection_group, focus_group=focus_group), + ) + ) + alleles[allele.vrs_digest] = AlleleIdentity( + level=allele.level, + hgvs=allele.hgvs, + clingen_allele_id=allele.clingen_allele_id, + is_focus=link.is_authoritative, + relation=relation.value if relation is not None else None, + derivation=derivation, + projection_of=projection_of, + ) + + return VariantDetail( + # TODO(#372) + urn=variant.urn or "", + scores=scores, + counts=counts, + classifications=classifications, + assay_level=assay_level, + target_hgvs=_submitted_assay_level_hgvs(variant, assay_level), + reference_hgvs=reference_hgvs, + assay_level_digest=assay_level_digest, + clingen_allele_id=clingen_allele_id, + pre_mapped=record.pre_mapped if record is not None else None, + post_mapped=authoritative.allele.post_mapped if authoritative is not None else None, + molecular_representation=molecular_representation, + mode=mode, + mapped=mapped_triple( + hgvs_c=measured_group.hgvs_c if measured_group is not None else None, + hgvs_g=measured_group.hgvs_g if measured_group is not None else None, + hgvs_p=measured_group.hgvs_p if measured_group is not None else None, + ), + alleles=alleles, + annotations=annotations, + is_current=superseding_score_set is None, + superseded_by_score_set=superseding_score_set.urn if superseding_score_set is not None else None, + ) diff --git a/src/mavedb/lib/variant_translations.py b/src/mavedb/lib/variant_translations.py index c24c5ed95..d4597b4e1 100644 --- a/src/mavedb/lib/variant_translations.py +++ b/src/mavedb/lib/variant_translations.py @@ -3,14 +3,24 @@ This module provides database operations for the variant_translations table, which stores relationships between protein allele (PA) and nucleotide allele (CA) ClinGen IDs. + +FROZEN (serving-only). The populate_variant_translations_for_score_set job that wrote this table was +retired in the #742 migration: the reverse-translation allele equivalence space (genomic/coding/protein +VRS alleles per variant, linked via MappingRecordAllele with HGVS on Allele) now covers PA<->CA +relationships without querying ClinGen. These helpers and the variant_translations table remain only to +serve existing old-model data; they are never written for new score sets and are dropped at read-cutover. """ from typing import cast +from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import CursorResult +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session +from mavedb.lib.vrs_utils import location_refgets +from mavedb.models.allele import Allele from mavedb.models.variant_translation import VariantTranslation @@ -39,3 +49,72 @@ def upsert_variant_translations(db: Session, translations: list[tuple[str, str]] created = result.rowcount existing = len(unique) - created return created, existing + + +class AlleleIdentityConflictError(ValueError): + """Raised when an allele's HGVS expression already exists under a different vrs_digest. + + An allele digest covers the refget of the sequence the allele sits on, and alleles deduplicate on + it. Two digests for one HGVS expression therefore mean two writers built the allele on different + sequences (most often by resolving its reference accession to different sequences), so the + copies would never merge. Nothing is wrong with the variant; the writers disagree about the + reference. + + To debug: compare the two refgets in the message with the sequence SeqRepo holds for the accession + (``SELECT * FROM seqalias WHERE alias = ''`` in the SeqRepo's aliases.sqlite3); check that + the mapper and the worker read the same SeqRepo (``SEQREPO_ROOT_DIR`` and ``HGVS_SEQREPO_DIR``) and + that dcd_mapping is current. Duplicates that already exist have to be removed, with the rows + that reference them, and the affected score sets remapped. Remove them first: this check refuses to + create an allele while another allele holds its HGVS. + """ + + +def _identity_conflict(existing: Allele, incoming: Allele) -> AlleleIdentityConflictError: + return AlleleIdentityConflictError( + f"Allele identity conflict for {incoming.hgvs}: an allele with this HGVS already " + f"exists under a different digest. Existing: id={existing.id} digest={existing.vrs_digest} " + f"refgets={location_refgets(existing.post_mapped or {})}. Incoming: digest={incoming.vrs_digest} " + f"refgets={location_refgets(incoming.post_mapped or {})}. Two writers disagree on the sequence " + "this allele sits on; see AlleleIdentityConflictError for how to debug this error." + ) + + +def get_or_create_allele(db: Session, allele_draft: Allele) -> Allele: + """Return the existing Allele matching ``allele_draft``'s vrs_digest, else add the draft. + + This is a get-or-create, not an upsert: a matching row is returned untouched, and on + a miss the draft is added to the session and flushed. The draft is never used to + update an existing row. + + :raise AlleleIdentityConflictError: if the draft's HGVS already exists under a different digest + + NOTE: This function does not commit; the caller is responsible for committing the session. + """ + existing = db.scalars(select(Allele).where(Allele.vrs_digest == allele_draft.vrs_digest)).one_or_none() + if existing is not None: + return existing + + hgvs = allele_draft.hgvs + if hgvs: + conflicting = db.scalars(select(Allele).where(Allele.hgvs == hgvs)).first() + if conflicting is not None: + raise _identity_conflict(conflicting, allele_draft) + + # Flushed inside a savepoint so a concurrent writer that slipped past the check above surfaces as + # the same error, through the unique index, without poisoning the caller's transaction. + try: + with db.begin_nested(): + db.add(allele_draft) + db.flush() + + except IntegrityError as e: + if "uq_alleles_hgvs" not in str(e.orig) or not hgvs: + raise + + conflicting = db.scalars(select(Allele).where(Allele.hgvs == hgvs)).first() + if conflicting is None: + raise + + raise _identity_conflict(conflicting, allele_draft) from e + + return allele_draft diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index 487d101aa..ba90886b8 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -1,54 +1,98 @@ import re from typing import Any, Optional +from mavedb.lib.hgvs import join_cis_phased_hgvs +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN, VARIANT_SCORE_DATA from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant + +def score_from_variant_data(data: Optional[Any]) -> Optional[float]: + """The canonical numeric score in a variant's ``data`` JSONB (``score_data.score``), or ``None``. + + The score column is required for every score set, but an individual variant may carry a null/NA + score. Returns ``None`` when the score is absent, null, or not coercible to a float; a numeric + string (``"1.5"``) coerces, but ``bool`` is rejected — a JSON ``true`` is not a score. Robust to + malformed JSONB: a non-mapping ``data`` or ``score_data`` yields ``None`` rather than raising. + Operates on the ``data`` mapping directly so it serves both ORM objects and bare ``Variant.data`` + column reads. + """ + if not isinstance(data, dict): + return None + + score_data = data.get(VARIANT_SCORE_DATA) + if not isinstance(score_data, dict): + return None + + value = score_data.get(REQUIRED_SCORE_COLUMN) + if value is None or isinstance(value, bool): + return None + + try: + return float(value) + except (TypeError, ValueError): + return None + + +def variant_score(variant: Variant) -> Optional[float]: + """The canonical numeric score of a variant (see :func:`score_from_variant_data`).""" + return score_from_variant_data(variant.data) + + HGVS_G_REGEX = re.compile(r"(^|:)g\.") HGVS_P_REGEX = re.compile(r"(^|:)p\.") -def hgvs_from_vrs_allele(allele: dict) -> str: +def hgvs_from_vrs_allele(allele: dict) -> Optional[str]: """ - Extract the HGVS notation from the VRS allele. + Extract the HGVS notation from the VRS allele, or None if it carries no expression. """ try: - # VRS 2.X - return allele["expressions"][0]["value"] + expressions = allele["expressions"] # VRS 2.X except KeyError: if "variation" in allele: raise ValueError("VRS 1.X format not supported.") # VRS 1.X. We don't want to allow this. - # return allele["variation"]["expressions"][0]["value"] - else: - raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + + # A valid VRS allele may simply carry no HGVS expression (None or empty) — e.g. a member of a + # cis-phased block. That is "no HGVS", not a crash. + if not expressions: + return None + return expressions[0]["value"] + +def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bool = False) -> Optional[str]: + """Extract a single HGVS string from a post-mapped VRS object. -def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: + Multi-variant blocks (Haplotype/CisPhasedBlock) are cis-phased, so their members combine + into one bracketed expression (``NC_…:g.[a;b]``) when ``combine_cis`` is set. It defaults + off because some consumers cannot yet handle a bracketed expression — notably ClinGen + submission, which has no single CAID for a multi-variant cis block (see + https://github.com/VariantEffect/mavedb-api/issues/764). + """ if not post_mapped_vrs: return None - if post_mapped_vrs["type"] == "Haplotype": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] - elif post_mapped_vrs["type"] == "CisPhasedBlock": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] - elif post_mapped_vrs["type"] == "Allele": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(post_mapped_vrs)] + if post_mapped_vrs["type"] in ("Haplotype", "CisPhasedBlock"): # type: ignore + members = post_mapped_vrs["members"] + elif post_mapped_vrs["type"] in ("Allele", "VariationDescriptor"): # type: ignore + members = [post_mapped_vrs] else: return None - if len(variations_hgvs) == 0: - return None - # raise ValueError(f"No variations found in variant {variant_urn}.") + member_hgvs = [hgvs_from_vrs_allele(allele) for allele in members] - # TODO (https://github.com/VariantEffect/mavedb-api/issues/468) In a future version, we will be able to generate - # a combined HGVS string for haplotypes and cis phased blocks directly from mapper output. - if len(variations_hgvs) > 1: + # No members, or a member carrying no HGVS expression — no single/combinable HGVS to return. + if not member_hgvs or any(h is None for h in member_hgvs): return None - # raise ValueError(f"Multiple variations found in variant {variant_urn}.") - return variations_hgvs[0] + hgvs_values: list[str] = [h for h in member_hgvs if h is not None] + if len(hgvs_values) > 1: + return join_cis_phased_hgvs(hgvs_values) if combine_cis else None + + return hgvs_values[0] def get_id_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: diff --git a/src/mavedb/lib/vep.py b/src/mavedb/lib/vep.py index a7d4e7b3c..b02c0ad34 100644 --- a/src/mavedb/lib/vep.py +++ b/src/mavedb/lib/vep.py @@ -1,187 +1,253 @@ -"""VEP (Variant Effect Predictor) library functions for functional consequence prediction.""" +"""VEP (Variant Effect Predictor) resolution and linking for molecular-consequence prediction. + +The resolution itself — VEP querying batched by transcript set, transcript-matched consequence +selection, the Variant Recoder fallback, and how recoded forms combine — is the shared +``variant_annotation.lib.vep`` orchestration, run over an Ensembl REST client. The api keeps no second +copy of that flow, so this pipeline and any other consumer (the lab CLI) resolve identically — the same +answer *and* the same flow for the same input. This module owns only the api-side concerns: running +that (synchronous) orchestration off the worker's event loop on a job-scoped client, the Ensembl release +lookup that version-keys the skip, and writing resolved consequences against deduplicated +:class:`Allele` rows. The worker job composes these; it holds none of the resolution mechanics itself. +""" import asyncio import functools import logging import os -from typing import Optional, Sequence +from datetime import date +from enum import Enum +from typing import Mapping, Optional, Sequence import requests - -from mavedb.lib.utils import request_with_backoff +from sqlalchemy import select +from sqlalchemy.orm import Session +from variant_annotation.lib.clients.ensembl import EnsemblRestClient +from variant_annotation.lib.vep import ( + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ConsequenceSource, + ReferenceSequence, + VepConfig, + VepInput, +) +from variant_annotation.lib.vep import resolve_consequences as resolve_consequences_kernel + +from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.models.enums.vep import VepConsequenceSource +from mavedb.models.vep_allele_consequence import VepAlleleConsequence logger = logging.getLogger(__name__) + ENSEMBL_API_URL = os.environ.get("ENSEMBL_API_URL", "https://rest.ensembl.org") -# List of all possible VEP consequences, in order from most to least severe -VEP_CONSEQUENCES = [ - "transcript_ablation", - "splice_acceptor_variant", - "splice_donor_variant", - "stop_gained", - "frameshift_variant", - "stop_lost", - "start_lost", - "transcript_amplification", - "inframe_insertion", - "inframe_deletion", - "missense_variant", - "disruptive_inframe_insertion", - "disruptive_inframe_deletion", - "protein_altering_variant", - "splice_region_variant", - "incomplete_terminal_codon_variant", - "start_retained", - "stop_retained", - "synonymous_variant", - "coding_sequence_variant", - "mature_miRNA_variant", - "5_prime_UTR_premature_start_codon_gain_variant", - "5_prime_UTR_variant", - "3_prime_UTR_variant", - "non_coding_transcript_exon_variant", - "non_coding_exon_variant", - "non_coding_transcript_variant", - "nc_transcript_variant", - "upstream_gene_variant", - "downstream_gene_variant", - "TFBS_ablation", - "TFBS_amplification", - "TF_binding_site_variant", - "regulatory_region_ablation", - "enhancer_ablation", - "regulatory_region_amplification", - "enhancer_amplification", - "regulatory_region_variant", - "feature_elongation", - "regulatory_region", - "TFBS", - "feature_truncation", - "exon_variant", - "disruptive_inframe_deletion", - "gene_variant", - "variant_affecting_coding_sequence_conservation", - "variant_affecting_genome_assembly_quality", - "variant_of_unknown_significance", - "sequence_variant", - "rare_amino_acid_variant", - "splice_region_variant", - "downstream_gene_variant", - "upstream_gene_variant", - "intron_variant", - "intergenic_variant", -] - - -async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str]]: - """Call the Variant Recoder API and return a mapping from input HGVS strings to genomic HGVS strings. - - Args: - missing_hgvs (Sequence[str]): List of HGVS strings to recode. - - Returns: - dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). - Returns an empty dict if Ensembl rejects the batch (e.g. 400 for - unrecognised identifiers) — callers treat missing entries as failures. + +class VepLinkVerdict(str, Enum): + """Per-allele outcome of a VEP linking run, returned for every allele whose status is decided. + + The single source of truth for what happened to an allele's consequence this run — the caller + derives annotation status from this, never by re-querying consequence state. An allele absent from + the map had no live consequence and resolved none this run (the caller reads that as "no result"). + + - ``CREATED`` — a new or changed consequence was created/superseded this run. + - ``UNCHANGED`` — a live consequence was re-confirmed (VEP resolved the same headline term). + - ``RETAINED_ON_ABSENCE`` — VEP found *nothing* this run, but a prior live consequence was held in + place rather than overwritten with a null. Kept distinct from ``UNCHANGED`` so the caller can + surface it: the allele still has a consequence, but the current Ensembl release did not confirm + it, which may mean a transient miss or a genuine upstream disappearance (transcript retirement). + """ + + CREATED = "created" + UNCHANGED = "unchanged" + RETAINED_ON_ABSENCE = "retained_on_absence" + + +async def get_ensembl_release() -> str: + """Return the current Ensembl release the REST API is serving, e.g. ``"116"`` (``/info/software``). + + A release is coordinated — software, transcript set, and consequence vocabulary all bump together + under one number — so this single value version-keys VEP results the way gnomAD keys on its data + version. The job stamps it on each consequence and skips alleles already live at the current release + (paired with ``RESOLVER_VERSION``). Delegates to the client's ``software_release`` so the release + lookup shares the resolution transport rather than being a second HTTP path. The client's synchronous + call runs off the event loop. Raises on failure: the version is load-bearing for the skip, so a job + that cannot determine it must abort rather than mis-version its writes. """ - headers = {"Content-Type": "application/json", "Accept": "application/json"} - # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor - # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( + with requests.Session() as session: + client = EnsemblRestClient(api_url=ENSEMBL_API_URL, session=session) + return await loop.run_in_executor(None, client.software_release) + + +async def resolve_consequences( + inputs: Sequence[VepInput], *, reference: Optional[ReferenceSequence] = None +) -> dict[str, ConsequenceResolution]: + """Resolve each input to a molecular consequence, keyed by its HGVS. One resolution per input. + + Delegates the full resolution flow to :func:`variant_annotation.lib.vep.resolve_consequences`, run + over an :class:`EnsemblRestClient` that satisfies both the VEP-lookup and Variant-Recoder ports: VEP + querying (batched by transcript set, resolved against each input's own transcript) and the Recoder + fallback for misses. The api keeps no second copy of that flow. + + ``reference`` is the optional transcript-reference port that lights up reference-identical detection: + a no-change input (an explicit ``c.=`` form, or a ``delins`` whose replacement bases equal what the + transcript already reads) resolves to ``REFERENCE_IDENTICAL`` rather than ``ABSENT``. It is consulted + only for inputs VEP and the Recoder both left unresolved — the wild-type controls and unparseables — + so it costs nothing for the inputs VEP resolves normally. When omitted, those inputs stay ``ABSENT``. + + The library call is synchronous; we run it in one executor call so the worker's event loop stays + free. It resolves serially (``VepConfig`` default ``max_workers=1``) to stay within Ensembl's + per-client rate limit. There is no per-input progress hook, so a caller reporting progress on a long + run chunks its inputs and awaits this once per chunk. + + The Ensembl client runs on a session we own and close when the call finishes, so its pooled + connections do not linger idle; the ``reference`` port's lifecycle is the caller's. Every input + appears in the result exactly once, with outcome ``RESOLVED`` / ``ABSENT`` / ``ERRORED`` — the caller + must retry ``ERRORED`` (unknown), never store it as a genuine empty. + """ + if not inputs: + return {} + + loop = asyncio.get_running_loop() + with requests.Session() as session: + client = EnsemblRestClient(api_url=ENSEMBL_API_URL, session=session) + resolutions = await loop.run_in_executor( None, functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/variant_recoder/human", - headers=headers, - json={"ids": list(missing_hgvs)}, - timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed + resolve_consequences_kernel, + inputs, + vep=client, + recoder=client, + reference=reference, + config=VepConfig(), ), ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier format) means the batch - # cannot be recoded. Return empty so callers can handle these missing entries. - logger.warning( - f"Variant Recoder returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(missing_hgvs)} HGVS strings — treating as no results.", - exc_info=exc, - ) - return {} - - data = response.json() - # request_with_backoff handles http errors, so no need to check response status - hgvs_to_genomic: dict[str, list[str]] = {} - for input_variant in data: - for variant_str, variant_data in input_variant.items(): - hgvs_string = variant_data.get("input") if isinstance(variant_data, dict) else None - if variant_str == "input" or not hgvs_string: - continue - genomic_strings = variant_data.get("hgvsg") if isinstance(variant_data, dict) else None - if genomic_strings: - for genomic_hgvs in genomic_strings: - if genomic_hgvs.startswith("NC_"): - hgvs_to_genomic.setdefault(hgvs_string, []).append(genomic_hgvs) - return hgvs_to_genomic - - -async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, Optional[str]]: - """Get VEP functional consequences for a batch of HGVS strings. - - Submits HGVS strings to the Ensembl VEP API and retrieves functional consequence - predictions. For any HGVS strings not found in the initial VEP response, attempts - to recode them using Variant Recoder and retries with VEP. - - Args: - hgvs_strings (Sequence[str]): List of HGVS strings to process (max 200 per call). - - Returns: - dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. - If no consequence found, maps to None. Returns an empty dict - if Ensembl rejects the batch (e.g. 400 for unrecognised - identifiers) — callers treat missing entries as needing Recoder - fallback or as failures. + return {resolution.input.hgvs: resolution for resolution in resolutions} + + +def _to_db_source(source: Optional[ConsequenceSource]) -> Optional[VepConsequenceSource]: + """Map the kernel's resolution source onto the api's persisted enum (identical values, distinct owners).""" + return VepConsequenceSource(source.value) if source is not None else None + + +def link_vep_consequences_to_alleles( + db: Session, + resolutions_by_allele_id: Mapping[int, ConsequenceResolution], + *, + source_version: str, + access_date: date, +) -> dict[int, VepLinkVerdict]: + """Store VEP consequences against deduplicated alleles, superseding only on a headline-term change. + + ``resolutions_by_allele_id`` maps each queried allele to the kernel's :class:`ConsequenceResolution`. + Only ``RESOLVED`` and ``ABSENT`` resolutions belong here — an ``ERRORED`` allele's failure is + recorded in the annotation event stream and it is retried, never linked, so passing one is a caller + bug and is ignored defensively. ``source_version`` is the Ensembl release the run resolved against. + Each allele holds at most one live :class:`VepAlleleConsequence`, handled per allele: + + - **unchanged** (live row already carries this headline consequence): advance ``source_version``, + ``resolver_version``, ``access_date``, and the provenance columns (``consequence_terms``/ + ``consequence_source``/``matched_transcript``) in place — no supersede. Supersede is keyed on the + headline term, not the release, resolver version, or provenance: a re-run that resolves the same + term must not fabricate a transaction-time boundary, and filling in a version/provenance column + for an already-held term (e.g. a pre-#772 row with NULL provenance, or a row predating + ``resolver_version``) is not a value change either. + - **new or changed** (no live row, or a different headline term): supersede keyed on ``allele_id`` + (retire the old, insert the successor stamped with this run's version + provenance). + - **absent this run** (``ABSENT`` outcome) with a live row: leave it in place — do not overwrite a + held consequence with a null result — and return :attr:`~VepLinkVerdict.RETAINED_ON_ABSENCE`. + This is loudly logged (per allele and in aggregate) because it can signal a genuine upstream + disappearance, not just a transient miss; the caller surfaces the aggregate in job metadata. + - **absent this run** with no live row: nothing to do; the allele is left out of the returned map. + + Does not commit. Returns a verdict per allele whose status is decided this run: + :attr:`~VepLinkVerdict.CREATED` (created/superseded), :attr:`~VepLinkVerdict.UNCHANGED` (a live + consequence re-confirmed), or :attr:`~VepLinkVerdict.RETAINED_ON_ABSENCE` (VEP found nothing, prior + consequence held). An allele absent from the returned map had no live row and resolved nothing — the + caller reads that as "no result". This is the single source of truth for per-allele status; callers + must not re-derive it from consequence state. """ - if len(hgvs_strings) > 200: - raise ValueError( - "VEP API can process a maximum of 200 HGVS strings per request. This function does not handle batching." + save_to_logging_context({"num_alleles_to_link_vep": len(resolutions_by_allele_id)}) + logger.debug(msg="Linking VEP consequences to alleles", extra=logging_context()) + + verdicts: dict[int, VepLinkVerdict] = {} + for allele_id, resolution in resolutions_by_allele_id.items(): + if resolution.outcome is ConsequenceOutcome.ERRORED: + # Failures are not this table's business; the caller records them in the event stream. + continue + + consequence = resolution.most_severe_consequence if resolution.outcome is ConsequenceOutcome.RESOLVED else None + + live = db.scalar( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) ) - headers = {"Content-Type": "application/json", "Accept": "application/json"} - result: dict[str, Optional[str]] = {} - - # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor - # keeps the event loop free during the full request + any retry wait time. - loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/vep/human/hgvs", - headers=headers, - json={"hgvs_notations": list(hgvs_strings)}, - timeout=60, # VEP can be slow for large batches. - ), + # VEP found nothing this run. Do not overwrite a held consequence with a null result; a retained + # consequence is RETAINED_ON_ABSENCE (surfaced), while no live row at all leaves the allele out + # of the map (the caller reads that as a no-result). + if consequence is None: + if live is not None: + logger.warning( + f"VEP returned NO consequence for allele {allele_id} (release {source_version}) that " + f"previously resolved to '{live.functional_consequence}'; retaining the prior value " + f"(not superseding). Investigate if this recurs — it can mean transcript retirement or " + f"an upstream Ensembl change, not a transient miss.", + extra=logging_context(), + ) + verdicts[allele_id] = VepLinkVerdict.RETAINED_ON_ABSENCE + + continue + + # Unchanged headline term: advance version/freshness and refresh provenance in place. + if live is not None and live.functional_consequence == consequence: + live.source_version = source_version + live.resolver_version = RESOLVER_VERSION + live.access_date = access_date + live.consequence_terms = list(resolution.consequence_terms) + live.consequence_source = _to_db_source(resolution.source) + live.matched_transcript = resolution.matched_transcript + verdicts[allele_id] = VepLinkVerdict.UNCHANGED + continue + + # New or changed consequence: retire any live row, insert the successor with full provenance. + VepAlleleConsequence.supersede_live_where( + db, + [ + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + consequence_terms=list(resolution.consequence_terms), + consequence_source=_to_db_source(resolution.source), + matched_transcript=resolution.matched_transcript, + source_version=source_version, + resolver_version=RESOLVER_VERSION, + access_date=access_date, + ) + ], + VepAlleleConsequence.allele_id == allele_id, ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier) means the batch cannot - # be resolved. Return empty so the callers can handle these missing entries. + verdicts[allele_id] = VepLinkVerdict.CREATED + + changed_allele_count = sum(1 for v in verdicts.values() if v is VepLinkVerdict.CREATED) + retained_on_absence_count = sum(1 for v in verdicts.values() if v is VepLinkVerdict.RETAINED_ON_ABSENCE) + save_to_logging_context( + {"changed_allele_count": changed_allele_count, "retained_on_absence_count": retained_on_absence_count} + ) + logger.info( + msg=f"Created or superseded {changed_allele_count} VEP allele consequences this run.", + extra=logging_context(), + ) + if retained_on_absence_count: + # Aggregate signal the job lifts into its outcome metadata: a large or growing count means VEP + # is now returning nothing for alleles that used to resolve — worth a look, not just a log line. logger.warning( - f"VEP returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(hgvs_strings)} HGVS strings — treating as no results.", - exc_info=exc, + msg=f"VEP returned no consequence for {retained_on_absence_count} allele(s) that previously had one; " + f"prior consequences were RETAINED, not superseded. A large or growing count may indicate " + f"transcript retirement or an upstream Ensembl data change.", + extra=logging_context(), ) - return result - - data = response.json() - for entry in data: - hgvs = entry.get("input") - most_severe_consequence = entry.get("most_severe_consequence") - if hgvs: - result[hgvs] = most_severe_consequence - - return result + return verdicts diff --git a/src/mavedb/lib/vrs.py b/src/mavedb/lib/vrs.py new file mode 100644 index 000000000..c5ee00ee1 --- /dev/null +++ b/src/mavedb/lib/vrs.py @@ -0,0 +1,134 @@ +"""Deserialization of stored ``post_mapped`` JSONB into GA4GH VRS objects. + +The mapping pipeline writes each allele's mapped representation as a VRS-shaped dict in +``Allele.post_mapped``. These helpers rehydrate that dict into strict ``ga4gh.vrs`` Pydantic models for the +Cat-VRS transit, the variant-detail envelope, and the VA-Spec annotation subjects. +""" + +from ga4gh.core.models import Extension +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + Expression, + LengthExpression, + LiteralSequenceExpression, + MolecularVariation, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, +) + + +def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: + """ + Converts a dictionary containing allelic mapping results into an Allele object. + + This function handles the possibility of an extra nesting level in early VRS 1.3 objects, + where Allele objects are contained within a `variation` property. If the `variation` key + is not present, the function assumes the dictionary itself represents the variation. + + Args: + allelic_mapping_results (dict): A dictionary containing allelic mapping results. + It may include a `variation` key or directly represent the variation. + + Returns: + Allele: An Allele object constructed from the provided mapping results. + + Raises: + KeyError: If required keys are missing from the input dictionary. + """ + + # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects + # are contained in a `variation` property. Although it's unlikely variants of this form + # will ever be exported in this format, we handle the possibility. + try: + variation = allelic_mapping_results["variation"] + except KeyError: + variation = allelic_mapping_results + + state_dict = variation["state"] + state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression + if state_dict.get("type") == "ReferenceLengthExpression": + state = ReferenceLengthExpression(**state_dict) + elif state_dict.get("type") == "LengthExpression": + state = LengthExpression(**state_dict) + elif state_dict.get("type") == "LiteralSequenceExpression": + state = LiteralSequenceExpression(**state_dict) + else: + raise ValueError( + f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " + "Update allele_from_mapped_variant_dictionary_result to handle this type." + ) + + # Mapping results were not guaranteed to be generated on this version of VRS. + # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on + # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid. In such cases, + # using model_validate() directly is not possible. + return Allele( + id=variation.get("id"), + state=state, + digest=variation.get("digest"), + location=SequenceLocation( + start=variation.get("location", {}).get("start"), + end=variation.get("location", {}).get("end"), + digest=variation.get("location", {}).get("digest"), + id=variation.get("location", {}).get("id"), + sequenceReference=SequenceReference( + name=variation.get("location", {}).get("sequenceReference", {}).get("name"), + refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), + ), + ), + extensions=[ + Extension( + id=extension.get("id"), + name=extension["name"], + description=extension.get("description"), + value=extension.get("value"), + ) + for extension in variation.get("extensions", []) + ], + expressions=[ + Expression( + id=expression.get("id"), + syntax=expression["syntax"], + syntax_version=expression.get("syntax_version"), + value=expression["value"], + ) + for expression in variation.get("expressions", []) + ], + ) + + +def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: + """ + Extracts a VRS (Variation Representation Specification) object from a stored ``post_mapped`` dict. + + This function processes a dictionary of mapping results and returns a VRS object, + which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object + returned depends on the "type" field in the input dictionary. + + Args: + mapping_results (dict): A dictionary containing the mapping results of a variant. + It must include a "type" key indicating the type of VRS object + ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). + If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also + include a "members" key containing a list of member alleles. + + Returns: + MolecularVariation: A VRS object representing the mapped variant. This will be + either a `CisPhasedBlock` containing its member variants or an `Allele` + derived from the mapping results. + + Raises: + KeyError: If required keys are missing from the `mapping_results` dictionary. + """ + if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": + return MolecularVariation( + # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) + CisPhasedBlock( + id=None, + members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], + ) + ) + + return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py new file mode 100644 index 000000000..3ad9525f8 --- /dev/null +++ b/src/mavedb/lib/vrs_utils.py @@ -0,0 +1,344 @@ +"""VRS allele identification helpers. + +Centralizes the digest-correctness invariant for GA4GH VRS alleles: the +``ga4gh_identify`` Merkle tree caches sub-object digests on the object after +first identification, so any subsequent mutation (refgetAccession swap, +normalization, state coercion) leaves a stale id unless the cached digests +are cleared first. All allele identification in dcd_mapping must route +through :func:`identify_allele` so the digest is always recomputed from +current content. +""" + +import logging +from collections.abc import Mapping +from itertools import cycle +from typing import Any + +from ga4gh.core import ga4gh_identify +from ga4gh.vrs.extras.translator import AlleleTranslator +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + Expression, + LiteralSequenceExpression, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, + Syntax, +) +from ga4gh.vrs.normalize import normalize + +from biocommons.seqrepo import SeqRepo + +from mavedb.lib.hgvs import extract_accession, split_cis_phased_hgvs +from mavedb.lib.logging.context import logging_context +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget +from mavedb.lib.vrs import vrs_object_from_mapped_variant + +logger = logging.getLogger(__name__) + +# HGVS type letter (``accession:g.``) → VRS Expression syntax. +_HGVS_SYNTAX_BY_TYPE = { + "g": Syntax.HGVS_G, + "c": Syntax.HGVS_C, + "p": Syntax.HGVS_P, + "n": Syntax.HGVS_N, + "m": Syntax.HGVS_M, + "r": Syntax.HGVS_R, +} + +# Accessions whose sequence SeqRepo is the authority for. Anything else (a target-sequence label, a +# ga4gh: digest) has no accession-to-sequence mapping to check an allele against. +CANONICAL_ACCESSION_PREFIXES = ("NC_", "NM_", "NR_", "NP_", "XM_", "XP_", "ENST", "ENSP") + + +class AlleleRefgetMismatchError(ValueError): + """Raised when an allele sits on a different sequence than SeqRepo holds for its accession.""" + + +def _hgvs_syntax(hgvs: str) -> Syntax: + """Map an HGVS string to its VRS Expression syntax via the type letter after the accession.""" + _, _, rest = hgvs.partition(":") + try: + return _HGVS_SYNTAX_BY_TYPE[rest[:1]] + except KeyError: + raise ValueError(f"Cannot determine HGVS syntax for {hgvs!r}") + + +def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: + """Convert HGVS variation description to VRS object. + + The AlleleTranslator is supplied by the caller and reused across calls. ga4gh's + AlleleTranslator opens a UTA connection lazily on first translate (via HgvsTools) + and holds it for its lifetime, so constructing one per call opens — and leaks — a + UTA connection per variant, exhausting the server's slot budget. Build one per + job/worker and pass it in. + + :param hgvs: MAVE-HGVS variation string + :param translator: caller-owned AlleleTranslator backed by a sequence/refget proxy + :return: Corresponding VRS allele as a Pydantic class + """ + # coerce tmp HGVS string into formally correct term + if hgvs.startswith("NC_") and ":c." in hgvs: + hgvs = hgvs.replace(":c.", ":g.") + + allele = translator.translate_from(hgvs, "hgvs", do_normalize=False) + + if ( + not isinstance(allele, Allele) + or not isinstance(allele.location, SequenceLocation) + or not isinstance(allele.location.start, int) + or not isinstance(allele.location.end, int) + or not isinstance(allele.state, LiteralSequenceExpression) + ): + raise ValueError + + return allele + + +def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Allele | CisPhasedBlock: + """Translate an HGVS expression — possibly a cis-phased multivariant — into a VRS object. + + Mirrors dcd_mapping's ``vrs_map._construct_vrs_allele``: each component HGVS is translated + to an Allele independently; a single component returns a bare Allele, while two or more are + wrapped in a CisPhasedBlock. The reverse-translation job emits bracketed genomic forms + (``g.[a;b]``) for non-adjacent codon components that ga4gh's AlleleTranslator cannot + translate directly, so splitting and recombining is the only way to represent them. + + The block's GA4GH digest is order-independent, so the same biological cis-phased set always + identifies to one ``ga4gh:CPB.`` digest and dedups to a single row regardless of component + ordering. + + Every component is normalized and re-identified through :func:`normalize_and_identify` + before use. ``translate_from`` is called with ``do_normalize=False`` and stamps ``id`` via + plain ``ga4gh_identify`` on the reused translator, so without this step a component carries a + non-canonical digest computed from the unnormalized object and from the Merkle tree's cached + sub-object digests — distinct biological variants can then collide onto one ``vrs_digest`` and + be merged by the digest-keyed ``get_or_create_allele``. Normalizing here also keeps RT digests + consistent with the mapper's, which is what lets the same allele dedup across sources. + + :param hgvs: a single- or cis-phased-multivariant HGVS string + :param translator: caller-owned AlleleTranslator reused across calls + :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set + """ + members = [] + for component in split_cis_phased_hgvs(hgvs): + allele = normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) + # Stamp the source HGVS as the allele's expression so post_mapped is self-describing. + # Mirrors the mapper's authoritative alleles. + allele.expressions = [Expression(syntax=_hgvs_syntax(component), value=component)] + members.append(allele) + + if len(members) == 1: + return members[0] + + block = CisPhasedBlock(members=members) # type: ignore[call-arg] + block.id = identify_variation(block) + return block + + +def identify_allele(allele: Allele) -> str: + """Clear cached digests and return a fresh GA4GH identifier for *allele*. + + ``ga4gh_identify`` is a Merkle-tree: it calls ``get_or_create_digest`` on + sub-objects, returning any cached value without recomputing. Clearing both + the location digest and the allele digest first ensures the id is always + derived from the current object content — not from a value set before a + refgetAccession mutation or normalization. + + ``id`` is recomputed as well: ``ga4gh_identify`` returns a non-empty ``id`` as-is + (its default ``in_place`` only fills an *empty* id), so an allele that already + carries one — e.g. stamped by an ``identify=True`` ``AlleleTranslator`` — would + otherwise keep its stale id even with the digests cleared. Use in_place="always" + to force it to be recomputed from the content-derived digest. + + The location needs identifying in its own right. ``ga4gh_identify`` writes only the + ``id`` of the object it is handed; for sub-objects it calls ``get_or_create_digest`` + and never ``get_or_create_ga4gh_identifier``. Clearing the location digest therefore + fixes what the *allele* id is derived from while leaving ``location.id`` holding + whatever was stamped before normalization — the allele id comes out correct and the + location it points at claims a digest belonging to the un-normalized span. Mirrors + ``dcd_mapping:vrs_utils.py::identify_allele``; both writers populate + ``alleles.post_mapped`` and have to agree byte-for-byte. + """ + if isinstance(allele.location, SequenceLocation): + allele.location.digest = None + allele.location.id = None + ga4gh_identify(allele.location, in_place="always") + + allele.digest = None + digest = ga4gh_identify(allele, in_place="always") + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for allele") # noqa: EM101 + + return digest + + +def identify_variation(variation: Allele | CisPhasedBlock) -> str: + """Clear cached digests and return a fresh GA4GH id for an Allele or CisPhasedBlock. + + Generalizes :func:`identify_allele` to cis-phased blocks. A block's Merkle digest is + derived from its members' digests, so a stale member digest would silently propagate into + the block id. Re-identifying every member through :func:`identify_allele` clears the member + and its location and restamps both, so a block never embeds a member whose ``id`` disagrees + with its own coordinates. + """ + if isinstance(variation, Allele): + return identify_allele(variation) + + for member in variation.members: + if isinstance(member, Allele): + member.id = identify_allele(member) + + variation.digest = None + digest = ga4gh_identify(variation, in_place="always") + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for variation") # noqa: EM101 + + return digest + + +def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: + """Normalize *allele* and stamp it with a freshly computed GA4GH digest. + + Pairs the finalize steps every VRS allele construction path needs. + Routing identification through :func:`identify_allele` (rather than + ``ga4gh_identify`` directly) is the invariant that protects against the + Merkle-tree's stale-digest behavior after mutation -- so any allele + construction site that bypasses this helper risks reintroducing the + stale-digest bug. + + Normalization can leave an indel as a ``ReferenceLengthExpression``; this coerces + it to a ``LiteralSequenceExpression`` so the result matches dcd_mapping's + authoritative alleles (``vrs_map._rle_to_lse``), which always store LSE. Without + the coercion the same biological indel hashes to two digests -- RLE here, LSE from + the mapper -- so reverse translation's regenerated genomic form fails to dedup + against the authoritative row and a duplicate allele is linked. + """ + allele = normalize(allele, data_proxy=data_proxy) + if isinstance(allele.state, ReferenceLengthExpression): + # Normalization yields an inlined SequenceLocation here, never an IRI reference. + assert isinstance(allele.location, SequenceLocation) + allele.state = _rle_to_lse(allele.state, allele.location, data_proxy) + + allele.id = identify_allele(allele) + return allele + + +def _rle_to_lse( + rle: ReferenceLengthExpression, location: SequenceLocation, data_proxy: Any +) -> LiteralSequenceExpression: + """Coerce a ReferenceLengthExpression to an equivalent LiteralSequenceExpression. + + Mirrors ``dcd_mapping:vrs_map.py::_rle_to_lse`` byte-for-byte so an allele built here + hashes identically to the mapper's authoritative allele for the same variant. Derives + the literal sequence by tiling the repeat subunit out to ``rle.length``. + """ + # A normalized indel location has an inlined SequenceReference and integer bounds; + # the IRI-reference and Range branches of these unions should never reach this helper. + assert isinstance(location.sequenceReference, SequenceReference) + assert isinstance(location.start, int) + assert isinstance(rle.length, int) + + sequence_id = location.sequenceReference.refgetAccession + start = location.start + end = start + rle.repeatSubunitLength + subsequence = data_proxy.get_sequence(f"ga4gh:{sequence_id}", start, end) + c = cycle(subsequence) + derived_sequence = "".join(next(c) for _ in range(rle.length)) + return LiteralSequenceExpression(sequence=derived_sequence) + + +def canonical_variation_document(document: Mapping[str, Any], *, subject: str) -> tuple[dict[str, Any], str]: + """Re-derive identity and canonical serialization for a variation arriving from outside. + + An identifier computed elsewhere is an *assertion*, not an identity: it may have been minted by a + different ``ga4gh.vrs`` version, or — the case that actually bit us — before a normalization step + this process does not control. ``AlleleTranslator`` stamps ``id`` at translation time by default + (``identify=True``), and ``ga4gh_identify``'s default ``in_place`` returns a non-empty ``id`` + unchanged, so a subsequent normalize-then-identify silently preserves the pre-normalization id. + Deletions and duplications are the variant classes whose normalized span differs, so they are the + ones that drift; a substitution normalizes to itself and its minted id stays correct by luck. + + Recomputing here makes the stored ``id`` a fact about the stored content rather than a claim + inherited from the producer, and returns one canonical serialization regardless of which writer + (or which library version) produced the input — so the same object stored twice is byte-identical. + + A disagreement is logged rather than raised: the incoming document is still usable, and refusing + the variant would lose a mapping over a repairable identifier. Silence is not an option either, + because a corrected identifier changes what the row deduplicates against. A disagreement in identifier + implies an upstream issue and should be investigated. + + :param document: the producer's serialized VRS Allele or CisPhasedBlock + :param subject: what this variation belongs to, for the warning message + :return: the canonical document and its GA4GH identifier + """ + variation = vrs_object_from_mapped_variant(dict(document)).root + if not isinstance(variation, Allele | CisPhasedBlock): + raise ValueError(f"Expected Allele or CisPhasedBlock for {subject}, got {type(variation).__name__}") + + identifier = identify_variation(variation) + + incoming = document.get("id") + if incoming and incoming != identifier: + logger.warning( + msg=( + f"VRS identifier mismatch for {subject}: producer supplied {incoming!r}, but the " + f"document's own content identifies as {identifier!r}. Storing the recomputed " + "identifier. This usually means the id was minted before normalization." + ), + extra=logging_context(), + ) + + return variation.model_dump(mode="json", exclude_none=True), identifier + + +def location_refgets(variation: Mapping[str, Any]) -> list[str]: + """Collect the refgetAccession of the variation's location, or of each member of a block.""" + members = variation.get("members") or [variation] + refgets = [] + for member in members: + reference = (member.get("location") or {}).get("sequenceReference") or {} + if reference.get("refgetAccession"): + refgets.append(reference["refgetAccession"]) + + return refgets + + +def verify_allele_refget(variation: Mapping[str, Any], hgvs: str, sr: SeqRepo, *, subject: str) -> None: + """Fail unless a variation sits on the sequence SeqRepo holds for its HGVS accession. + + An allele digest covers the refget of its sequence, and alleles deduplicate on that digest. Two + writers that resolve one accession to different sequences mint different digests for the same + variant, and the copies never merge (this happened with NM_007294.3 and NP_001346945.1). The + digest-consistency check in :func:`canonical_variation_document` cannot see it, because both + digests are internally consistent. Comparing against SeqRepo at insert time can. + + A mismatch raises. An accession SeqRepo cannot resolve, or resolves ambiguously, is logged at error + level without raising: the allele could not have been built on a sequence SeqRepo lacks, so it points + at a SeqRepo problem to alert on, not at this allele. + + :raise AlleleRefgetMismatchError: if the allele's refget differs from the accession's + """ + accession = extract_accession(hgvs) + if not accession.startswith(CANONICAL_ACCESSION_PREFIXES): + return + + try: + expected = resolve_refget(sr, accession) + except (SequenceNotFoundError, AmbiguousSequenceError) as e: + logger.error( + msg=f"Cannot verify the refget of {subject}: {e}", + extra=logging_context(), + ) + return + + for refget in location_refgets(variation): + if refget != expected: + msg = ( + f"Allele for {subject} sits on {refget}, but SeqRepo holds {expected} for {accession}. " + "The writer and SeqRepo disagree about this accession's sequence, so the allele would not " + "deduplicate against the same variant written elsewhere." + ) + logger.error(msg=msg, extra=logging_context()) + raise AlleleRefgetMismatchError(msg) diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index f23da015b..e4cf5139e 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -20,6 +20,16 @@ def annotation_pipeline_job_definitions( [(upstream_mapping_key, DependencyType.SUCCESS_REQUIRED)] if upstream_mapping_key else [] ) return [ + { + "key": "reverse_translate_variants_for_score_set", + "function": "reverse_translate_variants_for_score_set", + "type": JobType.MAPPED_VARIANT_ANNOTATION, + "params": { + "correlation_id": None, # Required param to be filled in at runtime + "score_set_id": None, # Required param to be filled in at runtime + }, + "dependencies": mapping_dep, + }, { "key": "submit_score_set_mappings_to_car", "function": "submit_score_set_mappings_to_car", @@ -29,7 +39,8 @@ def annotation_pipeline_job_definitions( "score_set_id": None, # Required param to be filled in at runtime "updater_id": None, # Required param to be filled in at runtime }, - "dependencies": mapping_dep, + "optional_params": {"force_reregister": False}, # Resubmit alleles that already hold a CAID + "dependencies": [("reverse_translate_variants_for_score_set", DependencyType.SUCCESS_REQUIRED)], }, { "key": "warm_clingen_cache", @@ -87,38 +98,14 @@ def annotation_pipeline_job_definitions( "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], }, { - "key": "populate_hgvs_for_score_set", - "function": "populate_hgvs_for_score_set", + "key": "populate_vep_for_score_set", + "function": "populate_vep_for_score_set", "type": JobType.MAPPED_VARIANT_ANNOTATION, "params": { "correlation_id": None, # Required param to be filled in at runtime "score_set_id": None, # Required param to be filled in at runtime }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], - }, - # VEP annotation is intentionally not scheduled: its consequences came from VEP's top-level - # most_severe_consequence, the worst call across all overlapping transcripts rather than the - # one the variant was mapped to. - # TODO(#772): Re-enable once VEP annotates Allele rows directly. - # { - # "key": "populate_vep_for_score_set", - # "function": "populate_vep_for_score_set", - # "type": JobType.MAPPED_VARIANT_ANNOTATION, - # "params": { - # "correlation_id": None, # Required param to be filled in at runtime - # "score_set_id": None, # Required param to be filled in at runtime - # }, - # "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], - # }, - { - "key": "populate_variant_translations_for_score_set", - "function": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], + "dependencies": [("reverse_translate_variants_for_score_set", DependencyType.SUCCESS_REQUIRED)], }, ] diff --git a/src/mavedb/lib/workflow/job_factory.py b/src/mavedb/lib/workflow/job_factory.py index 151cc3b57..4d6c4b243 100644 --- a/src/mavedb/lib/workflow/job_factory.py +++ b/src/mavedb/lib/workflow/job_factory.py @@ -51,6 +51,11 @@ def create_job_run( raise ValueError(f"Missing required param: {key}") job_params[key] = pipeline_params[key] + # Optional params take the pipeline's value when given and their declared default otherwise. + # Preset params above are fixed by the definition and never overridden. + for key, default in job_def.get("optional_params", {}).items(): + job_params[key] = pipeline_params.get(key, default) + job_run = JobRun( job_type=job_def["type"], job_function=job_def["function"], diff --git a/src/mavedb/lib/workflow/kickoff.py b/src/mavedb/lib/workflow/kickoff.py new file mode 100644 index 000000000..35f236bf2 --- /dev/null +++ b/src/mavedb/lib/workflow/kickoff.py @@ -0,0 +1,98 @@ +"""Create and enqueue a named pipeline for a single score set. + +The one shared kickoff seam: build the ``Pipeline`` / ``JobRun`` / ``JobDependency`` records via +:class:`PipelineFactory`, then enqueue the ``start_pipeline`` entrypoint in ARQ. Both the ad-hoc +``run_pipeline`` script and the batch ``enrich_backfilled_score_sets`` driver go through here so the +enqueue contract (entrypoint id + :func:`arq_job_id` + :func:`arq_queue_name`) lives in exactly one place. +""" + +from __future__ import annotations + +import datetime +import logging +from typing import Any, Optional + +from arq.connections import ArqRedis +from arq.jobs import Job +from sqlalchemy.orm import Session + +from mavedb.lib.logging.context import correlation_id_for_context, logging_context, save_to_logging_context +from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name + +logger = logging.getLogger(__name__) + + +async def enqueue_pipeline_for_score_set( + db: Session, + redis: ArqRedis, + *, + pipeline_name: str, + score_set: ScoreSet, + user: User, + extra_params: Optional[dict[str, Any]] = None, +) -> tuple[Pipeline, Optional[Job]]: + """Create the pipeline records for ``score_set`` and enqueue its ``start_pipeline`` entrypoint. + + Returns the created :class:`Pipeline` and the enqueued ARQ :class:`Job` (``None`` when ARQ + deduplicated an already-queued job with the same id). Raises ``KeyError`` for an unknown + ``pipeline_name`` and ``ValueError`` for an invalid pipeline definition, surfaced from + :meth:`PipelineFactory.create_pipeline`. If the ARQ enqueue itself raises, the just-created + pipeline records are discarded (via :meth:`PipelineFactory.discard_pipeline`) before the + exception is re-raised, so a failed enqueue never leaves an orphaned pipeline behind. + """ + correlation_id = ( + correlation_id_for_context() + or f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" + ) + params: dict[str, Any] = { + "correlation_id": correlation_id, + "score_set_id": score_set.id, + "updater_id": user.id, + } + if extra_params: + params.update(extra_params) + + factory = PipelineFactory(session=db) + pipeline, entrypoint = factory.create_pipeline( + pipeline_name=pipeline_name, creating_user=user, pipeline_params=params + ) + job_id = arq_job_id(entrypoint) + + try: + job = await redis.enqueue_job( + entrypoint.job_function, entrypoint.id, _job_id=job_id, _queue_name=arq_queue_name(entrypoint) + ) + except Exception: + logger.error( + "Failed to enqueue pipeline %s for score set %s; discarding pipeline %s", + pipeline_name, + score_set.urn, + pipeline.id, + extra=logging_context(), + ) + factory.discard_pipeline(pipeline) + raise + + save_to_logging_context({"pipeline_id": pipeline.id, "job_id": job_id}) + + if not job: + logger.info( + "Pipeline %s for score set %s already enqueued (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + else: + logger.info( + "Enqueued pipeline %s for score set %s (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + return pipeline, job diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 2f0d65b48..534ff0ec5 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -1,8 +1,11 @@ __all__ = [ "access_key", "acmg_classification", + "allele", + "calibration_control", "collection", "clinical_control", + "clinvar_allele_link", "controlled_keyword", "doi_identifier", "ensembl_identifier", @@ -10,13 +13,18 @@ "experiment", "experiment_set", "genome_identifier", + "gnomad_allele_link", "gnomad_variant", "job_dependency", "job_run", "legacy_keyword", "license", "mapped_variant", + "mapping_record", + "mapping_record_allele", + "mondo_term", "pipeline", + "projection_group", "publication_identifier", "published_variant", "variant_annotation_view", @@ -35,8 +43,12 @@ "taxonomy", "uniprot_identifier", "uniprot_offset", + "urn_redirect", "user", "variant_annotation_status", + "annotation_event", + "annotation_event_view", "variant", "variant_translation", + "vep_allele_consequence", ] diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py new file mode 100644 index 000000000..379cdd562 --- /dev/null +++ b/src/mavedb/models/allele.py @@ -0,0 +1,85 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Column, Date, Index, Integer, String, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.lib.hgvs import extract_accession + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + + +def _coalesce_hgvs(hgvs_g, hgvs_c, hgvs_p): + """SQL form of ``Allele.hgvs``.""" + return func.coalesce(hgvs_g, hgvs_c, hgvs_p) + + +class Allele(Base): + __tablename__ = "alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + + vrs_digest: Mapped[str] = Column(String, nullable=False) + level: Mapped[str] = Column(String(length=16), nullable=False) + + hgvs_g = Column(String, nullable=True) + hgvs_c = Column(String, nullable=True) + hgvs_p = Column(String, nullable=True) + + clingen_allele_id = Column(String, nullable=True) + post_mapped: Mapped[Optional[Any]] = Column(JSONB(none_as_null=True), nullable=True) + + created_at: Mapped[date] = Column(Date, nullable=False, default=date.today) + updated_at: Mapped[date] = Column(Date, nullable=False, default=date.today, onupdate=date.today) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the populated HGVS column (derived, not stored). + + Exactly one of hgvs_g/hgvs_c/hgvs_p is populated per allele, and the transcript is that + string's accession. Derived rather than stored so it cannot drift from the HGVS column + it duplicates. + """ + return extract_accession(self.hgvs or "") + + @hybrid_property + def hgvs(self) -> Optional[str]: + """The populated HGVS expression: whichever of hgvs_g/hgvs_c/hgvs_p is set (derived, not stored). + + None when none is, on the instance and in SQL alike (``coalesce`` yields NULL). + """ + return self.hgvs_g or self.hgvs_c or self.hgvs_p + + @hgvs.inplace.expression + @classmethod + def _hgvs_expression(cls): + return _coalesce_hgvs(cls.hgvs_g, cls.hgvs_c, cls.hgvs_p) + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(cls.hgvs, ":", 1) + + mapping_record_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="allele", + ) + + # Annotation links (VEP, gnomAD, ClinVar) deliberately carry no reverse collection here — they are + # one-directional annotation->Allele, navigated set-wise from the link tables, not from an Allele + # instance. Keep new annotation links one-directional unless a read path needs the navigation. + + __table_args__ = ( + UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + Index("ix_alleles_vrs_digest", "vrs_digest"), + Index("ix_alleles_level", "level"), + Index("ix_alleles_clingen_allele_id", "clingen_allele_id"), + # One row per HGVS expression. Two digests for the same expression mean two writers built the + # allele on different sequences, so the copies would never deduplicate. Fails at write time, in + # get_or_create_allele, where the cause is. See AlleleIdentityConflictError. + Index("uq_alleles_hgvs", _coalesce_hgvs(hgvs_g, hgvs_c, hgvs_p), unique=True), + ) diff --git a/src/mavedb/models/annotation_event.py b/src/mavedb/models/annotation_event.py new file mode 100644 index 000000000..5eba5ea4a --- /dev/null +++ b/src/mavedb/models/annotation_event.py @@ -0,0 +1,140 @@ +""" +SQLAlchemy model for the annotation event log. + +One append-only log spanning the whole pipeline (mapping, reverse translation, +annotation). Its subjects are exactly two persistent entities — ``Variant`` and +``Allele`` — selected by ``annotation_type`` via the polymorphic-subject CHECK. +Everything else the pipeline touches (``MappingRecord``, ``MappingRecordAllele``, +the external value tables) is a vehicle or resolution path, never a status subject. + +This is deliberately **not** a ``ValidTime`` table. A SCD-2 state table is a lossy +projection of an event log — it discards the skip/reconfirm/no-op events that are +the audit point. "Current" is derived (``DISTINCT ON (subject, annotation_type) +… id DESC``), never a stored ``current`` flag. +""" + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, Optional + +from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.mutable import MutableDict +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mavedb.db.base import Base +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason + +if TYPE_CHECKING: + from mavedb.models.allele import Allele + from mavedb.models.job_run import JobRun + from mavedb.models.score_set import ScoreSet + from mavedb.models.variant import Variant + +VARIANT_SUBJECT_TYPES = ( + AnnotationType.VRS_MAPPING.value, + AnnotationType.CROSS_LEVEL_TRANSLATION.value, + AnnotationType.VARIANT_TRANSLATION.value, + AnnotationType.LDH_SUBMISSION.value, +) +"""annotation_type values whose subject is the variant (variant_id set, allele_id null)""" + +ALLELE_SUBJECT_TYPES = ( + AnnotationType.CLINGEN_ALLELE_ID.value, + AnnotationType.GNOMAD_ALLELE_FREQUENCY.value, + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE.value, + AnnotationType.CLINVAR_CONTROL.value, + AnnotationType.MAPPED_HGVS.value, +) +"""annotation_type values whose subject is the allele (allele_id set, variant_id null)""" + + +def _sql_in_list(values: tuple[str, ...]) -> str: + return ", ".join(f"'{v}'" for v in values) + + +class AnnotationEvent(Base): + """An append-only event recording the *status* (not value) of one pipeline + observation about a ``Variant`` or an ``Allele``. + + The value (frequency, consequence, CAID, control) lives in the domain + ValidTime tables; this log records disposition + why + when. Reading the + domain tables alone cannot distinguish confirmed-absence from never-checked — + that gap is the whole reason the log exists. + + NOTE: JSONB ``event_metadata`` is tracked as a mutable object via MutableDict, + which only catches top-level mutations. Mutating a nested object + requires ``flag_modified(instance, "event_metadata")``. + """ + + __tablename__ = "annotation_event" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + annotation_type: Mapped[AnnotationType] = mapped_column(String(50), nullable=False) + + # Exactly one is set, per ck_annotation_event_subject. + variant_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("variants.id", ondelete="CASCADE"), nullable=True + ) + allele_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=True + ) + + disposition: Mapped[Disposition] = mapped_column(String(50), nullable=False) + + # Domain-specific code reusing in-code vocabularies (EventReason, plus MappingOutcome and RT + # skip_category for those two jobs); disposition is the public axis. + reason: Mapped[EventReason] = mapped_column(String(50), nullable=False) + + # gnomAD db_version / Ensembl release / ClinVar release / mapper version. + source_version: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + + # DB column is "metadata"; the attribute avoids the reserved Declarative name. + event_metadata: Mapped[Optional[Dict[str, Any]]] = mapped_column( + "metadata", MutableDict.as_mutable(JSONB), nullable=True + ) + + job_run_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("job_runs.id", ondelete="SET NULL"), nullable=True + ) + score_set_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("scoresets.id", ondelete="SET NULL"), nullable=True + ) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + # One-directional (no back-ref) — match the annotation-link convention. + variant: Mapped[Optional["Variant"]] = relationship("Variant") + allele: Mapped[Optional["Allele"]] = relationship("Allele") + job_run: Mapped[Optional["JobRun"]] = relationship("JobRun") + score_set: Mapped[Optional["ScoreSet"]] = relationship("ScoreSet") + + __table_args__ = ( + # Polymorphic subject: the type picks exactly one subject column. + CheckConstraint( + f"(annotation_type IN ({_sql_in_list(VARIANT_SUBJECT_TYPES)}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_sql_in_list(ALLELE_SUBJECT_TYPES)}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + # latest-per-allele / latest-per-variant (the DISTINCT ON … id DESC projections) + Index("ix_annotation_event_allele_type_id", "allele_id", "annotation_type", text("id DESC")), + Index("ix_annotation_event_variant_type_id", "variant_id", "annotation_type", text("id DESC")), + # version-keyed skip + Index("ix_annotation_event_allele_type_version", "allele_id", "annotation_type", "source_version"), + # audit by run + Index("ix_annotation_event_job_run_id", "job_run_id"), + # backs the score-set ON DELETE SET NULL cascade and score-set-scoped audit queries + Index("ix_annotation_event_score_set_id", "score_set_id"), + ) + + def __repr__(self) -> str: + subject = f"variant_id={self.variant_id}" if self.variant_id is not None else f"allele_id={self.allele_id}" + return ( + f"" + ) diff --git a/src/mavedb/models/annotation_event_view.py b/src/mavedb/models/annotation_event_view.py new file mode 100644 index 000000000..5c8547807 --- /dev/null +++ b/src/mavedb/models/annotation_event_view.py @@ -0,0 +1,96 @@ +"""``v_current_annotation_events`` — the current-state projection over the AnnotationEvent log. + +The log is append-only; "current" is derived, never stored. This view exposes the latest event per +``(subject, annotation_type)`` — one row per allele/variant per annotation type — so operators, BI, +and app code can read current status with a plain ``SELECT`` instead of re-deriving the +``DISTINCT ON`` everywhere. It mirrors the existing ``v_variant_annotations`` pattern. + +ClinVar is **multi-live**: an allele accumulates one live link per archival release, so its current +status is one row *per release*. The window partition folds ``source_version`` in **only** for +``clinvar_control`` (via a CASE that is constant-NULL for every other type, collapsing those back to +one row per subject+type). + +The subject is polymorphic — exactly one of ``variant_id`` / ``allele_id`` is set per row (enforced by +the log's CHECK), and the other is constant-NULL within a partition, so partitioning by both keys each +row to its real subject. + +There is deliberately **no ``score_set_id`` axis**: an allele-subject status (CAID, gnomAD, ClinVar, +VEP) is a *shared* allele-level fact, not a property of any one score set. A consumer that wants a +score set's annotation status resolves the score set's current alleles (and variants) through the live +mapping links — e.g. ``lib.clingen.alleles.get_alleles_for_score_set`` — and then looks those subjects +up here. Filtering by the *run's* score set would wrongly drop an allele last (re-)annotated by another +score set's run. The derived "current-for-variant" walk (resolving an allele-subject fact down to a +variant at the level a type keys on) is intentionally **not** built here — that is the deferred +consumer surface; see ``docs/design/allele-annotation-status.md``. +""" + +from sqlalchemy import case, func, select + +from mavedb.db.base import Base +from mavedb.db.view import view +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType + +signature = "v_current_annotation_events" + +# ClinVar is the lone multi-live type: split "current" by release. For every other type this is NULL, +# so the partition collapses to (allele_id, variant_id, annotation_type) — one current row per subject. +_clinvar_release_key = case( + (AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL.value, AnnotationEvent.source_version), + else_=None, +) + +_ranked = select( + AnnotationEvent.id.label("id"), + AnnotationEvent.annotation_type.label("annotation_type"), + AnnotationEvent.variant_id.label("variant_id"), + AnnotationEvent.allele_id.label("allele_id"), + AnnotationEvent.disposition.label("disposition"), + AnnotationEvent.reason.label("reason"), + AnnotationEvent.source_version.label("source_version"), + AnnotationEvent.event_metadata.label("event_metadata"), + AnnotationEvent.job_run_id.label("job_run_id"), + AnnotationEvent.created_at.label("created_at"), + func.row_number() + .over( + partition_by=[ + AnnotationEvent.allele_id, + AnnotationEvent.variant_id, + AnnotationEvent.annotation_type, + _clinvar_release_key, + ], + order_by=AnnotationEvent.id.desc(), + ) + .label("row_number"), +).subquery("ranked_annotation_events") + +definition = select( + _ranked.c.id, + _ranked.c.annotation_type, + _ranked.c.variant_id, + _ranked.c.allele_id, + _ranked.c.disposition, + _ranked.c.reason, + _ranked.c.source_version, + _ranked.c.event_metadata, + _ranked.c.job_run_id, + _ranked.c.created_at, +).where(_ranked.c.row_number == 1) + + +class CurrentAnnotationEventView(Base): + __table__ = view(signature, definition, materialized=False) + # Each surviving event id is unique across the view, so it is a valid mapping key for the + # otherwise-PK-less view (standard SQLAlchemy view-mapping idiom). + __mapper_args__ = {"primary_key": [__table__.c.id]} + + id = __table__.c.id + annotation_type = __table__.c.annotation_type + variant_id = __table__.c.variant_id + allele_id = __table__.c.allele_id + disposition = __table__.c.disposition + reason = __table__.c.reason + source_version = __table__.c.source_version + event_metadata = __table__.c.event_metadata + job_run_id = __table__.c.job_run_id + created_at = __table__.c.created_at diff --git a/src/mavedb/models/calibration_control.py b/src/mavedb/models/calibration_control.py new file mode 100644 index 000000000..144a1aeb0 --- /dev/null +++ b/src/mavedb/models/calibration_control.py @@ -0,0 +1,117 @@ +"""SQLAlchemy model for calibration controls. + +A *calibration control* is a variant whose clinical significance is independently +known (from a source outside MaveDB) and which was used as empirical ground truth +when deriving a calibration's score thresholds. Storing these controls lets a +clinician audit the evidence a calibration rests on before trusting its thresholds +for interpretation. +""" + +from __future__ import annotations + +from datetime import date +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import Column, Date, Enum, ForeignKey, Integer, UniqueConstraint, case, select +from sqlalchemy.orm import Mapped, column_property, relationship + +from mavedb.db.base import Base +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) + +if TYPE_CHECKING: + from mavedb.models.score_calibration import ScoreCalibration + from mavedb.models.user import User + from mavedb.models.variant import Variant + + +class CalibrationControl(Base): + """A variant with independently known clinical significance, used as ground truth for a calibration. + + Distinct from ``ScoreCalibrationFunctionalClassification`` and its ``variants`` + relationship, which record *bin membership* — the range a variant's functional score + happens to fall into. A ``CalibrationControl`` instead records *clinical ground truth*: + "this variant's pathogenicity is known from external evidence and was used to anchor + the calibration's thresholds." A control variant will usually also land in a bin, since + its score should agree with its known status, but the two relationships answer different + questions and are kept separate. + + Controls are constrained to variants in the calibration's own score set — a control + without a functional score is meaningless as calibration evidence. That constraint is + enforced at the API layer rather than in the schema (see issue #751). + """ + + __tablename__ = "calibration_controls" + __table_args__ = ( + # A variant may appear at most once per calibration and cannot be double counted. + UniqueConstraint("calibration_id", "variant_id", name="uq_calibration_controls_calibration_id_variant_id"), + ) + + id = Column(Integer, primary_key=True) + + # Deleting a calibration deletes its controls: a control has no meaning apart from the calibration it anchors. + # The DB-level cascade backstops the ORM ``delete-orphan`` cascade declared on ``ScoreCalibration.controls``. + calibration_id = Column( + Integer, ForeignKey("score_calibrations.id", ondelete="CASCADE"), nullable=False, index=True + ) + calibration: Mapped["ScoreCalibration"] = relationship("ScoreCalibration", back_populates="controls") + + variant_id = Column(Integer, ForeignKey("variants.id"), nullable=False, index=True) + variant: Mapped["Variant"] = relationship("Variant") + + # Which of the calibration's own functional classifications contains this control's variant by bin + # membership, or NULL when it lands under none. Computed live from the membership association rather + # than stored, so it can never drift from the clinical bin assignments it reports. Mirrors the + # ``variant_count`` correlated-subquery pattern on ScoreCalibrationFunctionalClassification. Ranges may + # overlap when one is 'not_specified' (see ScoreCalibrationBase.ranges_do_not_overlap), so a variant can + # fall in several bins. Classified bins never overlap each other, so at most one applies; it is + # preferred over a 'not_specified' bin (which asserts no classification), then the lowest id breaks + # any remaining tie, for a deterministic result. + functional_classification_id: Mapped[Optional[int]] = column_property( + select(ScoreCalibrationFunctionalClassification.id) + .where( + ScoreCalibrationFunctionalClassification.calibration_id == calibration_id, + score_calibration_functional_classification_variants_association_table.c.functional_classification_id + == ScoreCalibrationFunctionalClassification.id, + score_calibration_functional_classification_variants_association_table.c.variant_id == variant_id, + ) + .order_by( + case( + ( + ScoreCalibrationFunctionalClassification.functional_classification + == FunctionalClassification.not_specified, + 1, + ), + else_=0, + ), + ScoreCalibrationFunctionalClassification.id, + ) + .limit(1) + .correlate_except( + ScoreCalibrationFunctionalClassification, + score_calibration_functional_classification_variants_association_table, + ) + .scalar_subquery() + ) + + clinical_status = Column( + Enum(CalibrationControlStatus, native_enum=False, validate_strings=True, length=32), + nullable=False, + ) + + created_by_id = Column(Integer, ForeignKey("users.id"), index=True, nullable=False) + created_by: Mapped["User"] = relationship("User", foreign_keys="CalibrationControl.created_by_id") + modified_by_id = Column(Integer, ForeignKey("users.id"), index=True, nullable=False) + modified_by: Mapped["User"] = relationship("User", foreign_keys="CalibrationControl.modified_by_id") + creation_date = Column(Date, nullable=False, default=date.today) + modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + + def __repr__(self) -> str: # pragma: no cover - repr utility + return ( + f"" + ) diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index 0b989cb22..56ad491ca 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -1,40 +1,52 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Integer, String, UniqueConstraint -from sqlalchemy.orm import Mapped, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship from mavedb.db.base import Base from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table if TYPE_CHECKING: + from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.mapped_variant import MappedVariant -class ClinicalControl(Base): - __tablename__ = "clinical_controls" +class ClinvarControl(Base): + __tablename__ = "clinvar_controls" __table_args__ = ( UniqueConstraint( - "db_name", "db_identifier", "db_version", name="uq_clinical_controls_db_name_identifier_version" + "db_name", "db_identifier", "db_version", name="uq_clinvar_controls_db_name_identifier_version" ), ) - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - gene_symbol = Column(String, nullable=False, index=True) + gene_symbol: Mapped[str] = Column(String, nullable=False, index=True) - clinical_significance = Column(String, nullable=False) - clinical_review_status = Column(String, nullable=False) + clinical_significance: Mapped[str] = Column(String, nullable=False) + clinical_review_status: Mapped[str] = Column(String, nullable=False) - db_name = Column(String, nullable=False, index=True) - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False, index=True) + db_name: Mapped[str] = Column(String, nullable=False, index=True) + # ClinVar Allele ID (row level link). + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False, index=True) + + # ClinVar Variation ID (variation level link). + clinvar_variation_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen serving path: links to MappedVariant via the old association table (never written for new data). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=mapped_variants_clinical_controls_association_table, back_populates="clinical_controls", ) + + # New-model annotation links (one live link per allele per ClinVar release). + allele_links: Mapped[list["ClinvarAlleleLink"]] = relationship( + "ClinvarAlleleLink", + back_populates="clinvar_control", + ) diff --git a/src/mavedb/models/clinical_control_mapped_variant.py b/src/mavedb/models/clinical_control_mapped_variant.py index eabb7689a..b7fc5d84c 100644 --- a/src/mavedb/models/clinical_control_mapped_variant.py +++ b/src/mavedb/models/clinical_control_mapped_variant.py @@ -7,5 +7,5 @@ "mapped_variants_clinical_controls", Base.metadata, Column("mapped_variant_id", ForeignKey("mapped_variants.id"), primary_key=True), - Column("clinical_control_id", ForeignKey("clinical_controls.id"), primary_key=True), + Column("clinical_control_id", ForeignKey("clinvar_controls.id"), primary_key=True), ) diff --git a/src/mavedb/models/clinvar_allele_link.py b/src/mavedb/models/clinvar_allele_link.py new file mode 100644 index 000000000..941ee2f22 --- /dev/null +++ b/src/mavedb/models/clinvar_allele_link.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .clinical_control import ClinvarControl + + +class ClinvarAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a :class:`ClinvarControl` release. + + Replaces the frozen ``mapped_variants_clinical_controls`` association table for new-model writes. + A link is live while ``valid_to`` is NULL. Unlike gnomAD/VEP (one live result per allele), the partial + unique index is ``(allele_id, clinvar_control_id) WHERE valid_to IS NULL`` — **multi-live**: an allele + accumulates one live link per ClinVar release, because each release is a distinct, versioned + ``ClinvarControl`` assertion that stacks rather than supersedes. A link retires only on two theoretical + paths (archival data does not change): ClinVar drops the variant from a release (a re-run finds no data + for it), or the allele re-resolves to a *different* control within the same release — the job supersedes + that newest-wins to preserve one live link per (allele, release), since this index only enforces one live + link per (allele, control). + """ + + __tablename__ = "clinvar_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + clinvar_control_id: Mapped[int] = Column( + Integer, + ForeignKey("clinvar_controls.id", ondelete="RESTRICT"), + nullable=False, + ) + + # One-directional to Allele (no reverse collection there); back-ref on the ClinvarControl entity only. + allele: Mapped["Allele"] = relationship("Allele") + clinvar_control: Mapped["ClinvarControl"] = relationship("ClinvarControl", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_clinvar_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_clinvar_allele_links_clinvar_control_id", + "clinvar_control_id", + ), + # Multi-live: one live link per (allele, release). Each ClinVar release is a distinct + # ClinvarControl row, so different releases stack as independent live links rather than + # superseding. Only live rows participate in this constraint. + Index( + "uq_clinvar_allele_links_live", + "allele_id", + "clinvar_control_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/enums/annotation_layer.py b/src/mavedb/models/enums/annotation_layer.py deleted file mode 100644 index 2115f8e99..000000000 --- a/src/mavedb/models/enums/annotation_layer.py +++ /dev/null @@ -1,31 +0,0 @@ -from enum import Enum - - -class AnnotationLayer(str, Enum): - """Annotation layer for a variant mapping result. - - Mirrors the ``AnnotationLayer`` enum produced by the dcd-mapping QC API. - Values use full names so they round-trip readably through the database - column; the dcd-mapping payload uses short single-character codes - (``p`` / ``c`` / ``g``) which the worker translates via :func:`from_wire`. - """ - - protein = "protein" - cdna = "cdna" - genomic = "genomic" - - @classmethod - def from_wire(cls, code: str) -> "AnnotationLayer": - """Translate a dcd-mapping single-character code into an ``AnnotationLayer``.""" - try: - return _WIRE_TO_LAYER[code] - except KeyError as exc: - raise ValueError(f"Unknown annotation_level wire code: {code!r}") from exc - - -# Module-level so it doesn't get mistaken for an enum member. -_WIRE_TO_LAYER: dict[str, AnnotationLayer] = { - "p": AnnotationLayer.protein, - "c": AnnotationLayer.cdna, - "g": AnnotationLayer.genomic, -} diff --git a/src/mavedb/models/enums/annotation_type.py b/src/mavedb/models/enums/annotation_type.py index b1595347b..a3bbd1693 100644 --- a/src/mavedb/models/enums/annotation_type.py +++ b/src/mavedb/models/enums/annotation_type.py @@ -3,6 +3,7 @@ class AnnotationType(str, Enum): VRS_MAPPING = "vrs_mapping" + CROSS_LEVEL_TRANSLATION = "cross_level_translation" CLINGEN_ALLELE_ID = "clingen_allele_id" MAPPED_HGVS = "mapped_hgvs" VARIANT_TRANSLATION = "variant_translation" diff --git a/src/mavedb/models/enums/calibration_control_status.py b/src/mavedb/models/enums/calibration_control_status.py new file mode 100644 index 000000000..95c2fbb6e --- /dev/null +++ b/src/mavedb/models/enums/calibration_control_status.py @@ -0,0 +1,14 @@ +import enum + + +class CalibrationControlStatus(str, enum.Enum): + """Clinical significance of a calibration control variant. + + Deliberately restricted to the two-tier ACMG poles used to anchor a calibration's + thresholds. Intermediate tiers (VUS, likely pathogenic, likely benign) are excluded + by design: a control's value as empirical ground truth comes from a confident, binary + clinical call, not from a graded one. + """ + + pathogenic = "pathogenic" + benign = "benign" diff --git a/src/mavedb/models/enums/disposition.py b/src/mavedb/models/enums/disposition.py new file mode 100644 index 000000000..94438f292 --- /dev/null +++ b/src/mavedb/models/enums/disposition.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class Disposition(str, Enum): + """The stable, consumer-facing status axis of a variant event. + + Defined by *what the consumer may conclude* — not by the domain-specific + operation that produced it (that lives in ``reason``). + + - ``present`` — we hold the result / the step succeeded + - ``absent`` — the source or biology has nothing — an informative negative + - ``not_applicable`` — we could not ask — a pipeline/structural gap, not a statement about the source + - ``failed`` — errored, might retry; failure_category carries transient vs permanent + """ + + PRESENT = "present" + ABSENT = "absent" + NOT_APPLICABLE = "not_applicable" + FAILED = "failed" diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py new file mode 100644 index 000000000..503bc6b11 --- /dev/null +++ b/src/mavedb/models/enums/event_reason.py @@ -0,0 +1,58 @@ +from enum import Enum + + +class EventReason(str, Enum): + """Pipeline-wide vocabulary for a variant event's ``reason`` — the single field that says + *what happened*, spanning present / absent / not_applicable / failed. + + Reasons are shared across jobs wherever they mean the same thing (the ``annotation_type`` + already says *which* job), and job-specific only where a case is genuinely unique. One job + contributes its own pre-existing domain enum instead of duplicating here: ``mapping`` uses + ``MappingOutcome`` (mapped / intronic / no_protein_consequence / failed), which mirrors the + external dcd-mapping vocabulary. The full ``reason`` vocabulary is this enum plus that one. + + ``MIGRATED`` is the one exception to "reason means what happened": it deliberately does not + distinguish created/preexisting/reconfirmed etc., because a one-off data migration reconstructing + state from a legacy source cannot know which of those actually happened historically — only + ``disposition`` (present/absent/failed) is knowable there. Using it elsewhere would misrepresent a + live job's real observation as a backfill guess. + """ + + # spans present / absent / failed — a reconstructed observation, not a live run + MIGRATED = "migrated" # state reconstructed from a world lacking now required context; no per-run history + + # present — we hold the result / the step succeeded + CREATED = "created" # linked/registered this run (gnomAD, ClinVar, CAR) + PREEXISTING = "preexisting" # already held before this run (gnomAD, ClinVar, CAR) + RECONFIRMED = "reconfirmed" # re-verified unchanged (gnomAD, CAR force) + CORRECTED = "corrected" # replaced a stored value the source now assigns differently (CAR force) + SKIPPED = "skipped" # version-skip: already current at this source version (gnomAD, VEP) + SUPERSEDED = "superseded" # re-resolved within a release, newest wins (ClinVar) + SUBMITTED = "submitted" # LDH + TRANSLATED = "translated" # reverse translation + RECONFIRMATION_SKIPPED = "reconfirmation_skipped" # HGVS no longer buildable, existing CAID kept (CAR) + + # absent — the source or biology has nothing (informative negative) + NO_RECORD = "no_record" # source queried, returned nothing (gnomAD, ClinVar, VEP) + NO_CODING_TRANSCRIPT = "no_coding_transcript" # non-coding target has no protein consequence (RT) + + # not_applicable — we could not ask (structural gap) + NO_CAID = "no_caid" # no ClinGen allele id to key on (gnomAD, ClinVar) + NO_HGVS = "no_hgvs" # no HGVS to submit/resolve (CAR, VEP) + MULTI_VARIANT_CAID = "multi_variant_caid" # cis-block CAID cannot be used (ClinVar) + PROTEIN_LEVEL_ALLELE = ( + "protein_level_allele" # protein allele excluded from nucleotide-level annotation (ClinVar, gnomAD) + ) + NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) + NOT_TRANSLATABLE = "not_translatable" # protein consequence's edit type has no DNA equivalence class (RT) + CIS_PHASED_UNSUPPORTED = ( + "cis_phased_unsupported" # cis-phased multivariant assay HGVS; forward parser cannot read the allele list (RT) + ) + + # failed — errored + API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response, RT UTA outage) + SERVICE_REJECTED = "service_rejected" # external service refused the input (CAR) + MALFORMED_RESPONSE = "malformed_response" # unparseable/contractless response (CAR) + TRANSLATION_FAILED = "translation_failed" # all candidate HGVS failed translation (RT) + TRANSLATION_ERROR = "translation_error" # the translation engine errored (RT) + TRANSCRIPT_UNRESOLVED = "transcript_unresolved" # protein-coding target with no resolvable transcript (RT) diff --git a/src/mavedb/models/enums/measurement_relationship.py b/src/mavedb/models/enums/measurement_relationship.py new file mode 100644 index 000000000..cccf8bc17 --- /dev/null +++ b/src/mavedb/models/enums/measurement_relationship.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class MeasurementRelationship(str, Enum): + """How a measurement relates to the queried ClinGen id, by the measurement's *measured* level.""" + + direct = "direct" # assayed at the queried allele + protein_consequence = "protein_consequence" # protein measurement of a CA's consequence + nucleotide_encoding = "nucleotide_encoding" # an nt measurement of the protein apex's encoding diff --git a/src/mavedb/models/enums/sequence_level.py b/src/mavedb/models/enums/sequence_level.py new file mode 100644 index 000000000..62875f99f --- /dev/null +++ b/src/mavedb/models/enums/sequence_level.py @@ -0,0 +1,48 @@ +from enum import Enum + + +class SequenceLevel(str, Enum): + """The molecular sequence level of a variant representation: genomic DNA, coding DNA, or protein. + + A single, duty-neutral closed set reused across several columns that each carry a different + semantic meaning: the level a variant was *assayed* at (``assay_level``), the level dcd-mapping + *aligned* it at (``alignment_level``), and the level of a stored allele (``level``). + + Values use full names so they round-trip readably through the database column; the dcd-mapping + payload uses short single-character codes (``p`` / ``c`` / ``g``) which the worker translates via + :func:`from_wire`. + """ + + protein = "protein" + cdna = "cdna" + genomic = "genomic" + + @classmethod + def from_wire(cls, code: str) -> "SequenceLevel": + """Translate a dcd-mapping single-character code into a ``SequenceLevel``.""" + try: + return _WIRE_TO_LEVEL[code] + except KeyError as exc: + raise ValueError(f"Unknown sequence level wire code: {code!r}") from exc + + @classmethod + def nucleotide_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are nucleotide (genomic or coding).""" + return {cls.genomic, cls.cdna} + + @classmethod + def protein_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are protein (protein only).""" + return {cls.protein} + + +# Module-level so it doesn't get mistaken for an enum member. +_WIRE_TO_LEVEL: dict[str, SequenceLevel] = { + "p": SequenceLevel.protein, + "c": SequenceLevel.cdna, + "g": SequenceLevel.genomic, +} + + +NUCLEOTIDE_LEVELS = {level.value for level in SequenceLevel.nucleotide_levels()} +PROTEIN_LEVELS = {level.value for level in SequenceLevel.protein_levels()} diff --git a/src/mavedb/models/enums/vep.py b/src/mavedb/models/enums/vep.py new file mode 100644 index 000000000..a45384083 --- /dev/null +++ b/src/mavedb/models/enums/vep.py @@ -0,0 +1,28 @@ +from enum import Enum + + +class VepConsequenceSource(str, Enum): + """Which resolution path produced a stored VEP consequence — provenance, not severity. + + Persisted on every :class:`~mavedb.models.vep_allele_consequence.VepAlleleConsequence` so a stored + consequence can be audited without re-querying Ensembl, and so a consumer can tell a + transcript-specific call from a cross-transcript one, see #772. + + - ``transcript`` — read from the ``transcript_consequences`` entry matching the allele's own + transcript. The trustworthy case: the consequence applies to the transcript the allele lives on. + - ``most_severe`` — VEP's top-level ``most_severe_consequence``, the worst call across *every* + transcript overlapping the position. Used only when no transcript could be matched (a genomic + allele carrying no transcript, or a transcript absent from VEP's set). Lower confidence: it + regularly reports a consequence that might not apply to the transcript of interest. + - ``reference_identical`` — the allele describes no sequence change (a wild-type control); no VEP + call was made. Recorded distinctly so it is never mistaken for ``synonymous_variant`` or a null. + + Values mirror ``variant_annotation.lib.vep.ConsequenceSource``: the library computes the source, the + api owns this closed set. The two are kept in sync by + :func:`mavedb.lib.vep.link_vep_consequences_to_alleles`, which maps one to the other by value at the + write boundary. + """ + + transcript = "transcript" + most_severe = "most_severe" + reference_identical = "reference_identical" diff --git a/src/mavedb/models/gnomad_allele_link.py b/src/mavedb/models/gnomad_allele_link.py new file mode 100644 index 000000000..75294ba66 --- /dev/null +++ b/src/mavedb/models/gnomad_allele_link.py @@ -0,0 +1,60 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .gnomad_variant import GnomADVariant + + +class GnomadAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a gnomAD variant. + + Replaces the frozen ``gnomad_variants_mapped_variants`` association table for new-model writes. + A link is live while ``valid_to`` is NULL; a gnomAD version bump retires the live row and inserts + a successor rather than deleting, so prior-version frequency links remain queryable point-in-time. + The partial unique index enforces **a single live link per allele** — gnomAD frequency is one + current value, so a new version supersedes the old (unlike ClinVar, which keeps one live link per + release). This matches the VEP consequence shape, not the ClinVar control shape. + """ + + __tablename__ = "gnomad_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + gnomad_variant_id: Mapped[int] = Column( + Integer, + ForeignKey("gnomad_variants.id", ondelete="RESTRICT"), + nullable=False, + ) + + allele: Mapped["Allele"] = relationship("Allele") + gnomad_variant: Mapped["GnomADVariant"] = relationship("GnomADVariant", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_gnomad_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_variant_id", + ), + # At most one live link per allele. A version bump supersedes (retires the old, inserts the + # new) rather than accumulating per-version live links; superseded rows stay for point-in-time + # queries. Only the live row participates in this constraint. + Index( + "uq_gnomad_allele_links_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/gnomad_variant.py b/src/mavedb/models/gnomad_variant.py index 0f4a00a50..c0904f614 100644 --- a/src/mavedb/models/gnomad_variant.py +++ b/src/mavedb/models/gnomad_variant.py @@ -1,37 +1,46 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional -from sqlalchemy import Column, Date, Integer, Float, String -from sqlalchemy.orm import Mapped, relationship +from sqlalchemy import Column, Date, Float, Integer, String +from sqlalchemy.orm import Mapped, mapped_column, relationship from mavedb.db.base import Base from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: + from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapped_variant import MappedVariant class GnomADVariant(Base): __tablename__ = "gnomad_variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - db_name = Column(String, nullable=False) - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False) + db_name: Mapped[str] = Column(String, nullable=False) + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False) - allele_count = Column(Integer, nullable=False) - allele_number = Column(Integer, nullable=False) - allele_frequency = Column(Float, nullable=False) + allele_count: Mapped[int] = Column(Integer, nullable=False) + allele_number: Mapped[int] = Column(Integer, nullable=False) + allele_frequency: Mapped[float] = mapped_column(Float, nullable=False) - faf95_max = Column(Float, nullable=True) - faf95_max_ancestry = Column(String, nullable=True) + faf95_max: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + faf95_max_ancestry: Mapped[Optional[str]] = mapped_column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen association to the old MappedVariant model — read by serving for existing data, never + # written for new score sets (which link through ``allele_links`` instead). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=gnomad_variants_mapped_variants_association_table, back_populates="gnomad_variants", ) + + # Valid-time links to deduplicated alleles (new-model writes). + allele_links: Mapped[list["GnomadAlleleLink"]] = relationship( + "GnomadAlleleLink", + back_populates="gnomad_variant", + ) diff --git a/src/mavedb/models/job_run.py b/src/mavedb/models/job_run.py index 877eeab02..b079a1757 100644 --- a/src/mavedb/models/job_run.py +++ b/src/mavedb/models/job_run.py @@ -70,6 +70,14 @@ class JobRun(Base): progress_total: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) progress_message: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + # Liveness heartbeat. `onupdate=func.now()` bumps this on *any* commit that dirties the row, + # so every progress checkpoint (and every lifecycle transition) refreshes it with zero per-job + # code. The stalled-job sweeper treats a RUNNING job whose heartbeat has gone stale as wedged, + # which distinguishes a healthy long-running job (still committing progress) from a dead one. + progress_updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now() + ) + # Correlation for tracing correlation_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) @@ -98,6 +106,8 @@ class JobRun(Base): Index("ix_job_runs_created_at", "created_at"), Index("ix_job_runs_correlation_id", "correlation_id"), Index("ix_job_runs_status_scheduled", "status", "scheduled_at"), + # Supports the stalled-job sweeper's RUNNING heartbeat query (status + heartbeat staleness). + Index("ix_job_runs_status_progress_updated", "status", "progress_updated_at"), CheckConstraint( "status IN ('pending', 'queued', 'running', 'succeeded', 'failed', 'errored', 'cancelled', 'skipped')", name="ck_job_runs_status_valid", diff --git a/src/mavedb/models/mapped_variant.py b/src/mavedb/models/mapped_variant.py index b35f1bbb4..10914f778 100644 --- a/src/mavedb/models/mapped_variant.py +++ b/src/mavedb/models/mapped_variant.py @@ -7,11 +7,11 @@ from mavedb.db.base import Base from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: - from .clinical_control import ClinicalControl + from .clinical_control import ClinvarControl from .gnomad_variant import GnomADVariant from .target_gene_mapping import TargetGeneMapping from .variant import Variant @@ -44,7 +44,7 @@ class MappedVariant(Base): # Per-mapping QC annotations from dcd-mapping ScoreAnnotation. alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=True, ) at_mismatched_locus = Column(Boolean, nullable=True) @@ -61,8 +61,8 @@ class MappedVariant(Base): hgvs_c = Column(String, nullable=True) hgvs_p = Column(String, nullable=True) - clinical_controls: Mapped[list["ClinicalControl"]] = relationship( - "ClinicalControl", + clinical_controls: Mapped[list["ClinvarControl"]] = relationship( + "ClinvarControl", secondary=mapped_variants_clinical_controls_association_table, back_populates="mapped_variants", ) diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py new file mode 100644 index 000000000..d0838ab27 --- /dev/null +++ b/src/mavedb/models/mapping_record.py @@ -0,0 +1,132 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, ForeignKeyConstraint, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime +from mavedb.lib.hgvs import extract_accession +from mavedb.models.enums.sequence_level import SequenceLevel + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + from .projection_group import ProjectionGroup + from .target_gene_mapping import TargetGeneMapping + from .variant import Variant + + +class MappingRecord(ValidTime, Base): + __tablename__ = "mapping_records" + + id: Mapped[int] = Column(Integer, primary_key=True) + + variant_id: Mapped[int] = Column(Integer, nullable=False) + # Denormalized from the variant so an RLS policy can check the score set directly; delegating + # through variants costs ~700 ms per lookup at production size (#833). The composite foreign key + # below keeps it equal to the variant's score set. + score_set_id: Mapped[int] = Column(Integer, nullable=False) + variant: Mapped["Variant"] = relationship("Variant", back_populates="mapping_records") + + # Digest of the pre-mapped (assayed-level) VRS representation, indexed for + # cross-score-set dedup lookups. + vrs_digest = Column(String, nullable=True) + pre_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + + # Level at which the variant was *assayed* — distinct from alignment_level (QC). Same closed + # SequenceLevel set as its sibling below. The prod CHECK (ck_mapping_records_assay_level_valid) + # predates this column being typed as an Enum; native_enum=False keeps it a VARCHAR(16), so + # declaring the Enum here needs no migration — it self-documents the model and gives the + # metadata-built (test) schema the same guard. + assay_level: Mapped[SequenceLevel] = Column( + # Distinct `name` so the generated CHECK doesn't collide with alignment_level's in this same + # table (both default to "sequencelevel", which Postgres rejects as a duplicate per table). + Enum( + SequenceLevel, + name="assay_level", + create_constraint=True, + length=16, + native_enum=False, + validate_strings=True, + ), + nullable=False, + ) + hgvs_assay_level = Column(String, nullable=True) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the assay-level HGVS (derived, not stored). + + Derived rather than stored so it cannot drift from hgvs_assay_level, which already + carries the accession. + """ + return extract_accession(self.hgvs_assay_level or "") + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(cls.hgvs_assay_level, ":", 1) + + mapping_api_version = Column(String, nullable=False) + # Domain data: the date the mapping was performed (from the dcd-mapping run), surfaced to + # users — distinct from valid_from, which is when this version became the live record. + mapped_date = Column(Date, nullable=False, default=date.today) + vrs_version = Column(String, nullable=True) + + # valid_from / valid_to and the derived `current` come from ValidTime: a re-map retires the + # prior live record (closes its valid_to) and inserts a new live one, so mapping history is + # retained and point-in-time queries are a single predicate. + + # Per-mapping QC fields from dcd-mapping. + alignment_level = Column( + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), + nullable=True, + ) + at_mismatched_locus = Column(Boolean, nullable=True) + near_gap = Column(Boolean, nullable=True) + + target_gene_mapping_id = Column(Integer, ForeignKey("target_gene_mappings.id"), index=True, nullable=True) + target_gene_mapping: Mapped[Optional["TargetGeneMapping"]] = relationship( + "TargetGeneMapping", back_populates="mapping_records" + ) + + allele_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="mapping_record", + cascade="all, delete-orphan", + ) + + projection_groups: Mapped[list["ProjectionGroup"]] = relationship( + "ProjectionGroup", + back_populates="mapping_record", + cascade="all, delete-orphan", + ) + + # Retiring a record retires its live allele links and projection groups (the measured group and any + # candidate groups a reverse-translation run attached). See ValidTime.__retire_cascade__. + __retire_cascade__ = ("allele_links", "projection_groups") + + __table_args__ = ( + ForeignKeyConstraint( + [variant_id, score_set_id], + ["variants.id", "variants.scoreset_id"], + name="fk_mapping_records_variant_score_set", + link_to_name=True, + ondelete="CASCADE", + ), + # Target of mapping_record_alleles' composite foreign key. + Index("uq_mapping_records_id_score_set", id, score_set_id, unique=True), + Index("ix_mapping_records_vrs_digest", "vrs_digest"), + Index("ix_mapping_records_variant_id", "variant_id"), + # At most one live mapping record per variant — promotes to the database the invariant the + # mapping job enforces in app code (it retires the prior live record before inserting a new + # one). Superseded versions remain with a closed valid_to for point-in-time queries. + Index( + "uq_mapping_records_current", + "variant_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py new file mode 100644 index 000000000..b19823067 --- /dev/null +++ b/src/mavedb/models/mapping_record_allele.py @@ -0,0 +1,87 @@ +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import Boolean, Column, ForeignKey, ForeignKeyConstraint, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .mapping_record import MappingRecord + from .projection_group import ProjectionGroup + + +class MappingRecordAllele(ValidTime, Base): + __tablename__ = "mapping_record_alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + mapping_record_id: Mapped[int] = Column(Integer, nullable=False) + # Denormalized from the mapping record for the RLS policy, like MappingRecord.score_set_id (#833). + score_set_id: Mapped[int] = Column(Integer, nullable=False) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + # True for the assay's actual measured allele (from vrs_map); False for translator-derived + # alleles. Lives on the link, not Allele, since the same allele can be authoritative for one + # mapping record and derived for another. + is_authoritative: Mapped[bool] = Column(Boolean, nullable=False, default=False) + # The projection group this allele describes one level of. The authoritative link's group is the record's + # measured group. NULL for an allele outside any group (the protein apex of reverse translation). + projection_group = Column(Integer, nullable=True) + + mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") + allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") + # Read-only: writers set ``projection_group`` directly. A group row is never edited in place (a new + # description is a new row, and its links are re-pointed), so this resolves to the group the link was + # written against at any instant. + group: Mapped[Optional["ProjectionGroup"]] = relationship( + "ProjectionGroup", + primaryjoin="foreign(MappingRecordAllele.projection_group) == ProjectionGroup.id", + viewonly=True, + ) + + __table_args__ = ( + ForeignKeyConstraint( + [mapping_record_id, score_set_id], + ["mapping_records.id", "mapping_records.score_set_id"], + name="fk_mapping_record_alleles_mapping_record_score_set", + ondelete="CASCADE", + ), + # MATCH SIMPLE: a NULL projection_group is not checked. + ForeignKeyConstraint( + [projection_group, mapping_record_id], + ["projection_groups.id", "projection_groups.mapping_record_id"], + name="fk_mapping_record_alleles_projection_group", + ondelete="CASCADE", + ), + Index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_id", + ), + Index( + "ix_mapping_record_alleles_allele_id", + "allele_id", + ), + Index("ix_mapping_record_alleles_projection_group", "projection_group"), + # At most one live link per (mapping_record, allele); retired links (valid_to set) are + # exempt so history is preserved. + Index( + "uq_mapping_record_alleles_live", + "mapping_record_id", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + # At most one live authoritative link per mapping record. lib/score_set_variants relies on + # this 1:1 invariant (inner join on the authoritative link); a second one would silently + # duplicate the variant and double-count its score. + Index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_id", + unique=True, + postgresql_where=text("is_authoritative AND valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/mondo_term.py b/src/mavedb/models/mondo_term.py new file mode 100644 index 000000000..7059e1b0a --- /dev/null +++ b/src/mavedb/models/mondo_term.py @@ -0,0 +1,39 @@ +"""SQLAlchemy model for MONDO disease terms. + +A ``MondoTerm`` is a single controlled-vocabulary entry (a MONDO code + its label) referenced by +calibrations as their disease/disorder context. The columns are deliberately the generic coding +columns (``system``/``code``/``system_version``/``label``) rather than MONDO-specific ones: this table +is the narrow, first consumer of a shared concept model, and matching the column shape of +``controlled_keywords`` (and the eventual ``mappable_concepts`` table) keeps a later promotion additive +rather than a reshape. On the wire a term is served as a GA4GH ``MappableConcept``. +""" + +from datetime import date +from typing import Optional + +from sqlalchemy import Date, Integer, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from mavedb.db.base import Base + + +class MondoTerm(Base): + __tablename__ = "mondo_terms" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + # The coding: ``code`` is the MONDO CURIE (e.g. "MONDO:0015263"), ``system`` its terminology + # identifier. ``system`` is constant for MONDO today but stored explicitly so a promotion to a + # multi-system concept table needs no backfill. ``UNIQUE(system, code)`` dedupes shared terms. + code: Mapped[str] = mapped_column(String, nullable=False) + system: Mapped[str] = mapped_column(String, nullable=False) + system_version: Mapped[Optional[str]] = mapped_column(String, nullable=True) + label: Mapped[str] = mapped_column(String, nullable=False) + + creation_date: Mapped[date] = mapped_column(Date, nullable=False, default=date.today) + modification_date: Mapped[date] = mapped_column(Date, nullable=False, default=date.today, onupdate=date.today) + + __table_args__ = (UniqueConstraint("system", "code", name="uq_mondo_terms_system_code"),) + + def __repr__(self) -> str: # pragma: no cover - repr utility + return f"" diff --git a/src/mavedb/models/projection_group.py b/src/mavedb/models/projection_group.py new file mode 100644 index 000000000..89e2a93cf --- /dev/null +++ b/src/mavedb/models/projection_group.py @@ -0,0 +1,54 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKeyConstraint, Index, Integer, String +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .mapping_record import MappingRecord + + +class ProjectionGroup(ValidTime, Base): + """One variant described at the coding, genomic and protein levels, within one mapping record. + + A record has one *measured* group, the one its authoritative allele link points at, holding the + projections of the variant it measured; and one *candidate* group per pair reverse translation + constructs from the protein consequence. Allele links join a group through + ``MappingRecordAllele.projection_group``. + + The HGVS strings are the group's description, independent of whether each level has an allele: an + intronic or UTR-spanning coding projection has no VRS allele (the transcript has no sequence there) + but is still the variant's coding coordinate. Where a level does have an allele, its HGVS repeats the + string here. + """ + + __tablename__ = "projection_groups" + + id: Mapped[int] = Column(Integer, primary_key=True) + mapping_record_id: Mapped[int] = Column(Integer, nullable=False) + + # Denormalized from the mapping record for the RLS policy, like MappingRecord.score_set_id (#833). + score_set_id: Mapped[int] = Column(Integer, nullable=False) + + hgvs_c = Column(String, nullable=True) + hgvs_g = Column(String, nullable=True) + # The protein consequence, shared by every group of a record. Null where the variant has none (intronic, + # UTR) or it could not be determined. + hgvs_p = Column(String, nullable=True) + + mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="projection_groups") + + __table_args__ = ( + ForeignKeyConstraint( + [mapping_record_id, score_set_id], + ["mapping_records.id", "mapping_records.score_set_id"], + name="fk_projection_groups_mapping_record_score_set", + ondelete="CASCADE", + ), + Index("ix_projection_groups_mapping_record_id", "mapping_record_id"), + # Target of mapping_record_alleles' composite foreign key, which keeps a link inside its own + # record's groups. + Index("uq_projection_groups_id_mapping_record", "id", "mapping_record_id", unique=True), + ) diff --git a/src/mavedb/models/published_variant.py b/src/mavedb/models/published_variant.py index eff749bd7..503d3fa30 100644 --- a/src/mavedb/models/published_variant.py +++ b/src/mavedb/models/published_variant.py @@ -1,25 +1,29 @@ -from sqlalchemy import select, join +from sqlalchemy import join, select from mavedb.db.view import MaterializedView, view - +from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.mapped_variant import MappedVariant - signature = "published_variants_materialized_view" + +# One row per (published variant, mapping-record version). The LEFT OUTER JOIN to mapping_records +# carries NO liveness filter on purpose: every record version (live and superseded) is retained so +# the statistics layer can count either the current mappings or the full history via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). Unmapped variants stay in the view with a +# NULL ``mapping_record_id``. definition = ( select( Variant.id.label("variant_id"), Variant.urn.label("variant_urn"), - MappedVariant.id.label("mapped_variant_id"), + MappingRecord.id.label("mapping_record_id"), ScoreSet.id.label("score_set_id"), ScoreSet.urn.label("score_set_urn"), ScoreSet.published_date.label("published_date"), - MappedVariant.current.label("current_mapped_variant"), + MappingRecord.valid_to.is_(None).label("current_mapping_record"), ) .select_from( - join(Variant, MappedVariant, Variant.id == MappedVariant.variant_id, isouter=True).join( + join(Variant, MappingRecord, Variant.id == MappingRecord.variant_id, isouter=True).join( ScoreSet, ScoreSet.id == Variant.score_set_id ) ) @@ -38,8 +42,8 @@ class PublishedVariantsMV(MaterializedView): variant_id = __table__.c.variant_id variant_urn = __table__.c.variant_urn - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id score_set_id = __table__.c.score_set_id score_set_urn = __table__.c.score_set_urn published_date = __table__.c.published_date - current_mapped_variant = __table__.c.current_mapped_variant + current_mapping_record = __table__.c.current_mapping_record diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index 38ce1f286..2b7ecc3cc 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -5,17 +5,19 @@ from datetime import date from typing import TYPE_CHECKING -from sqlalchemy import Boolean, Column, Date, Float, ForeignKey, Integer, String +from sqlalchemy import Boolean, Column, Date, Float, ForeignKey, Integer, String, func, select from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.associationproxy import AssociationProxy, association_proxy -from sqlalchemy.orm import Mapped, relationship +from sqlalchemy.orm import Mapped, column_property, relationship from mavedb.db.base import Base from mavedb.lib.urns import generate_calibration_urn +from mavedb.models.calibration_control import CalibrationControl from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_calibration_publication_identifier import ScoreCalibrationPublicationIdentifierAssociation if TYPE_CHECKING: + from mavedb.models.mondo_term import MondoTerm from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.score_set import ScoreSet from mavedb.models.user import User @@ -25,7 +27,7 @@ class ScoreCalibration(Base): __tablename__ = "score_calibrations" # TODO#544: Add a partial unique index to enforce only one primary calibration per score set. - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), nullable=True, default=generate_calibration_urn, unique=True, index=True) score_set_id = Column(Integer, ForeignKey("scoresets.id"), nullable=False) @@ -33,7 +35,7 @@ class ScoreCalibration(Base): title = Column(String, nullable=False) research_use_only = Column(Boolean, nullable=False, default=False) - primary = Column(Boolean, nullable=False, default=False) + primary: Mapped[bool] = Column(Boolean, nullable=False, default=False) investigator_provided: Mapped[bool] = Column(Boolean, nullable=False, default=False) private = Column(Boolean, nullable=False, default=True) notes = Column(String, nullable=True) @@ -41,12 +43,41 @@ class ScoreCalibration(Base): baseline_score = Column(Float, nullable=True) baseline_score_description = Column(String, nullable=True) + # MONDO-coded disease/disorder context. Non-nullable and defaulted to the generic "disease or disorder" term, + # so every calibration carries a concept and consumers never branch on null. + disease_term_id = Column(Integer, ForeignKey("mondo_terms.id"), nullable=False, index=True) + disease_term: Mapped["MondoTerm"] = relationship("MondoTerm") + + # Submitter's affirmation that the control data contains no PHI. Tristate on purpose: None = not yet addressed, + # False = explicitly declined, True = affirmed. Controls may be stored only while this is True; withdrawing it + # deletes them (see lib.score_calibrations.require_phi_affirmation, #752). + controls_not_phi = Column(Boolean, nullable=True) + functional_classifications: Mapped[list["ScoreCalibrationFunctionalClassification"]] = relationship( "ScoreCalibrationFunctionalClassification", back_populates="calibration", cascade="all, delete-orphan", ) + # Ground-truth controls anchoring this calibration's thresholds. Separate from the per-bin ``variants`` on each + # functional classification, which record score-range membership rather than known clinical significance + # (see ``CalibrationControl``). + controls: Mapped[list["CalibrationControl"]] = relationship( + "CalibrationControl", + back_populates="calibration", + cascade="all, delete-orphan", + ) + + # Efficient count via correlated subquery — lets list/collection responses report how many + # controls a calibration has without loading the control rows. Mirrors + # ``ScoreCalibrationFunctionalClassification.variant_count``. + controls_count: Mapped[int] = column_property( + select(func.count(CalibrationControl.id)) + .where(CalibrationControl.calibration_id == id) + .correlate_except(CalibrationControl) + .scalar_subquery() + ) + publication_identifier_associations: Mapped[list[ScoreCalibrationPublicationIdentifierAssociation]] = relationship( "ScoreCalibrationPublicationIdentifierAssociation", back_populates="score_calibration", diff --git a/src/mavedb/models/score_calibration_functional_classification.py b/src/mavedb/models/score_calibration_functional_classification.py index b9cdabc49..9adc46a6e 100644 --- a/src/mavedb/models/score_calibration_functional_classification.py +++ b/src/mavedb/models/score_calibration_functional_classification.py @@ -22,6 +22,15 @@ class ScoreCalibrationFunctionalClassification(Base): + """A labeled functional-score range (or class) within a calibration, plus the variants that fall in it. + + The ``variants`` relationship records *bin membership* — which variants' functional scores land + in this range. That is distinct from ``CalibrationControl`` (see ``models/calibration_control.py``), + which records variants with independently known clinical significance used as ground truth for the + calibration. A variant can be both, but the two are stored separately because they answer different + questions: "where did this score land?" versus "what is this variant's known clinical status?". + """ + __tablename__ = "score_calibration_functional_classifications" id = Column(Integer, primary_key=True) diff --git a/src/mavedb/models/score_calibration_functional_classification_variant_association.py b/src/mavedb/models/score_calibration_functional_classification_variant_association.py index 61f074bd2..e9475f798 100644 --- a/src/mavedb/models/score_calibration_functional_classification_variant_association.py +++ b/src/mavedb/models/score_calibration_functional_classification_variant_association.py @@ -1,6 +1,6 @@ """SQLAlchemy association table for variants belonging to functional classifications.""" -from sqlalchemy import Column, ForeignKey, Table +from sqlalchemy import Column, ForeignKey, Index, Table from mavedb.db.base import Base @@ -11,4 +11,5 @@ "functional_classification_id", ForeignKey("score_calibration_functional_classifications.id"), primary_key=True ), Column("variant_id", ForeignKey("variants.id"), primary_key=True), + Index("ix_score_calib_func_classification_variants_variant_id", "variant_id"), ) diff --git a/src/mavedb/models/score_set.py b/src/mavedb/models/score_set.py index ccdcb73d5..1c71bf6b5 100644 --- a/src/mavedb/models/score_set.py +++ b/src/mavedb/models/score_set.py @@ -74,7 +74,7 @@ class ScoreSet(Base): __tablename__ = "scoresets" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) # TODO(#372) urn = Column(String(64), default=generate_temp_urn, index=True, nullable=True, unique=True) @@ -101,6 +101,8 @@ class ScoreSet(Base): nullable=True, ) processing_errors = Column(JSONB, nullable=True) + # Binned summary for small distribution charts; see mavedb.lib.score_distribution. Null until computed. + score_distribution = Column(JSONB, nullable=True) data_usage_policy = Column(String, nullable=True) num_variants = Column(Integer, nullable=False, default=0) @@ -124,7 +126,14 @@ class ScoreSet(Base): # TODO Standardize on US or GB spelling for licenc/se. licence_id = Column(Integer, ForeignKey("licenses.id"), index=True, nullable=False) license: Mapped["License"] = relationship("License") - superseded_score_set_id = Column("replaces_id", Integer, ForeignKey("scoresets.id"), index=True, nullable=True, unique=True,) + superseded_score_set_id = Column( + "replaces_id", + Integer, + ForeignKey("scoresets.id"), + index=True, + nullable=True, + unique=True, + ) superseded_score_set: Mapped[Optional["ScoreSet"]] = relationship( "ScoreSet", uselist=False, diff --git a/src/mavedb/models/target_gene.py b/src/mavedb/models/target_gene.py index 671a4380f..5419c3478 100644 --- a/src/mavedb/models/target_gene.py +++ b/src/mavedb/models/target_gene.py @@ -23,10 +23,10 @@ class TargetGene(Base): __tablename__ = "target_genes" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) name = Column(String, nullable=False) - category = Column( + category: Mapped[TargetCategory] = Column( Enum(TargetCategory, create_constraint=True, length=32, native_enum=False, validate_strings=True), nullable=False, ) diff --git a/src/mavedb/models/target_gene_mapping.py b/src/mavedb/models/target_gene_mapping.py index 417f249d4..a128d9e14 100644 --- a/src/mavedb/models/target_gene_mapping.py +++ b/src/mavedb/models/target_gene_mapping.py @@ -19,23 +19,24 @@ from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel if TYPE_CHECKING: from mavedb.models.mapped_variant import MappedVariant + from mavedb.models.mapping_record import MappingRecord from mavedb.models.target_gene import TargetGene class TargetGeneMapping(Base): __tablename__ = "target_gene_mappings" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - target_gene_id = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) + target_gene_id: Mapped[int] = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) target_gene: Mapped["TargetGene"] = relationship("TargetGene", back_populates="target_gene_mappings") alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=False, ) preferred = Column(Boolean, nullable=False, default=False, server_default="false") @@ -72,6 +73,10 @@ class TargetGeneMapping(Base): # ``modification_date``, which track this row's lifecycle in MaveDB. mapped_date = Column(Date, nullable=True) + # The mapping job (JobRun) that produced this row. Nullable: rows written before this column, + # and rows whose job was pruned (ON DELETE SET NULL), carry no run anchor. + job_run_id = Column(Integer, ForeignKey("job_runs.id", ondelete="SET NULL"), nullable=True, index=True) + creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) @@ -80,4 +85,9 @@ class TargetGeneMapping(Base): back_populates="target_gene_mapping", ) + mapping_records: Mapped[list["MappingRecord"]] = relationship( + "MappingRecord", + back_populates="target_gene_mapping", + ) + __table_args__ = (Index("ix_target_gene_mappings_target_alignment", "target_gene_id", "alignment_level"),) diff --git a/src/mavedb/models/urn_redirect.py b/src/mavedb/models/urn_redirect.py new file mode 100644 index 000000000..5cd24f38a --- /dev/null +++ b/src/mavedb/models/urn_redirect.py @@ -0,0 +1,36 @@ +""" +SQLAlchemy model for URNs that publication has retired. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from mavedb.db.base import Base + + +class UrnRedirect(Base): + """ + Records that a record's URN was replaced, and by what. + + Publishing a dataset overwrites the ``tmp:`` URN it was created with, so every link already + shared to the unpublished record stops resolving. One row is written here per URN publication + retires, and requests naming a retired URN are forwarded to its replacement. + + Only experiment sets, experiments and score sets get rows. A variant's URN is + ``{score_set_urn}#{n}``, so forwarding replaces the retired URN wherever it appears in a request + path and a variant follows its score set without a row of its own. + """ + + __tablename__ = "urn_redirects" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + old_urn: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + new_urn: Mapped[str] = mapped_column(String(64), nullable=False) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + def __repr__(self) -> str: + return f"" diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index 59b6e729c..773589788 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -1,26 +1,28 @@ from datetime import date -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, List, Optional -from sqlalchemy import Column, Date, ForeignKey, Integer, String +from sqlalchemy import Column, ColumnElement, Date, ForeignKey, Index, Integer, String, cast, func from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base if TYPE_CHECKING: from .mapped_variant import MappedVariant + from .mapping_record import MappingRecord from .score_set import ScoreSet class Variant(Base): __tablename__ = "variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), index=True, nullable=True, unique=True) data = Column(JSONB, nullable=False) - score_set_id = Column("scoreset_id", Integer, ForeignKey("scoresets.id"), index=True, nullable=False) + score_set_id: Mapped[int] = Column("scoreset_id", Integer, ForeignKey("scoresets.id"), index=True, nullable=False) # TODO examine if delete-orphan is necessary, explore cascade score_set: Mapped["ScoreSet"] = relationship(back_populates="variants") @@ -35,5 +37,30 @@ class Variant(Base): back_populates="variant", cascade="all, delete-orphan" ) + mapping_records: Mapped[List["MappingRecord"]] = relationship( + back_populates="variant", cascade="all, delete-orphan" + ) + # Bidirectional relationship with ScoreCalibrationFunctionalClassification is left # purposefully undefined for performance reasons. + + __table_args__ = ( + # Target of mapping_records' composite (variant_id, score_set_id) foreign key. The records + # carry their score set so an RLS policy can check it without joining through variants (#833). + Index("uq_variants_id_scoreset", id, score_set_id, unique=True), + ) + + @hybrid_property + def variant_number(self) -> Optional[int]: + """The integer after '#' in the URN, which orders a score set's variants.""" + return int(self.urn.split("#")[1]) if self.urn else None + + @variant_number.inplace.expression + @classmethod + def _variant_number_expression(cls) -> ColumnElement[int]: + # Order by this expression rather than restating it: the index below serves only an identical one. + return cast(func.split_part(cls.urn, "#", 2), Integer) + + +# Serves ordering a score set's variants by number, so deep pages of a large score set skip the full sort. +Index("ix_variants_scoreset_number", Variant.score_set_id, Variant.variant_number, Variant.id) diff --git a/src/mavedb/models/variant_annotation_status.py b/src/mavedb/models/variant_annotation_status.py index f39c47a64..357d6a6e6 100644 --- a/src/mavedb/models/variant_annotation_status.py +++ b/src/mavedb/models/variant_annotation_status.py @@ -93,7 +93,7 @@ class VariantAnnotationStatus(Base): # FK index for job_run_id — needed for CASCADE deletes on job_runs Index("ix_variant_annotation_status_job_run_id", "job_run_id"), CheckConstraint( - "annotation_type IN ('vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", + "annotation_type IN ('vrs_mapping', 'cross_level_translation', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", name="ck_variant_annotation_type_valid", ), CheckConstraint( diff --git a/src/mavedb/models/variant_annotation_view.py b/src/mavedb/models/variant_annotation_view.py index c7c77d6f8..6b6e52963 100644 --- a/src/mavedb/models/variant_annotation_view.py +++ b/src/mavedb/models/variant_annotation_view.py @@ -2,13 +2,89 @@ from mavedb.db.base import Base from mavedb.db.view import view -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence signature = "v_variant_annotations" +# All allele-derived columns are correlated scalar subqueries keyed on the outer MappingRecord, so the +# outer query's FROM stays Variant / ScoreSet / MappingRecord / VariantAnnotationStatus and never +# references Allele / MappingRecordAllele directly. That keeps the grain at one row per +# (variant, annotation_type) — matching the legacy view — and, crucially, avoids ``aliased()`` at +# module import (which would force full mapper configuration before every model is registered, breaking +# a bare ``import`` of this module by Alembic). + + +def _auth_allele(column): + """Scalar subquery: ``column`` of the outer record's live **authoritative** (measured) allele. + + Source of the single-valued ``clingen_allele_id``. The authoritative link is 1:1 with the record + (the mapping job upholds one live authoritative link per record), so this resolves a single allele. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _auth_vep(column): + """Scalar subquery: ``column`` of the live VEP consequence on the record's authoritative allele.""" + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.valid_to.is_(None)), + ) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _record_hgvs(level: str, column): + """Scalar subquery: the canonical HGVS at ``level`` for the outer record. + + Each mapped level (genomic / cdna / protein) is a distinct deduplicated :class:`Allele` linked to + the record. This constructs a flat ``hgvs_g`` / ``hgvs_c`` / ``hgvs_p`` triple by pivoting the record's + live allele links by level — yielding ``NULL`` where the record has no allele at that level. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + Allele.level == level, + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +# Flat operator/CLI convenience view: one row per (variant, current annotation_type). definition = ( select( Variant.urn.label("variant_urn"), @@ -16,18 +92,17 @@ Variant.hgvs_nt, Variant.hgvs_pro, Variant.hgvs_splice, - MappedVariant.id.label("mapped_variant_id"), - MappedVariant.clingen_allele_id, - MappedVariant.hgvs_assay_level, - MappedVariant.hgvs_g, - MappedVariant.hgvs_c, - MappedVariant.hgvs_p, - MappedVariant.vep_functional_consequence, - MappedVariant.vep_access_date, - MappedVariant.mapped_date, - MappedVariant.mapping_api_version, - MappedVariant.vrs_version, - MappedVariant.error_message.label("mapping_error"), + MappingRecord.id.label("mapping_record_id"), + _auth_allele(Allele.clingen_allele_id).label("clingen_allele_id"), + MappingRecord.hgvs_assay_level, + _record_hgvs("genomic", Allele.hgvs_g).label("hgvs_g"), + _record_hgvs("cdna", Allele.hgvs_c).label("hgvs_c"), + _record_hgvs("protein", Allele.hgvs_p).label("hgvs_p"), + _auth_vep(VepAlleleConsequence.functional_consequence).label("vep_functional_consequence"), + _auth_vep(VepAlleleConsequence.access_date).label("vep_access_date"), + MappingRecord.mapped_date, + MappingRecord.mapping_api_version, + MappingRecord.vrs_version, VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status.label("annotation_status"), VariantAnnotationStatus.failure_category, @@ -40,10 +115,9 @@ ) .select_from(Variant) .join(ScoreSet, ScoreSet.id == Variant.score_set_id) - .outerjoin( - MappedVariant, - and_(MappedVariant.variant_id == Variant.id, MappedVariant.current == True), # noqa: E712 - ) + # The variant's live mapping record. live-ness rides the ON clause (never a WHERE) so the outer + # join is preserved for unmapped variants — a WHERE would reject the null-record row. + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) .outerjoin( VariantAnnotationStatus, and_(VariantAnnotationStatus.variant_id == Variant.id, VariantAnnotationStatus.current == True), # noqa: E712 @@ -59,7 +133,7 @@ class VariantAnnotationView(Base): hgvs_nt = __table__.c.hgvs_nt hgvs_pro = __table__.c.hgvs_pro hgvs_splice = __table__.c.hgvs_splice - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id clingen_allele_id = __table__.c.clingen_allele_id hgvs_assay_level = __table__.c.hgvs_assay_level hgvs_g = __table__.c.hgvs_g @@ -70,7 +144,6 @@ class VariantAnnotationView(Base): mapped_date = __table__.c.mapped_date mapping_api_version = __table__.c.mapping_api_version vrs_version = __table__.c.vrs_version - mapping_error = __table__.c.mapping_error annotation_type = __table__.c.annotation_type annotation_status = __table__.c.annotation_status failure_category = __table__.c.failure_category diff --git a/src/mavedb/models/variant_translation.py b/src/mavedb/models/variant_translation.py index a50d36ce2..36aa0dfdf 100644 --- a/src/mavedb/models/variant_translation.py +++ b/src/mavedb/models/variant_translation.py @@ -6,6 +6,10 @@ class VariantTranslation(Base): + """FROZEN (serving-only). Written by the retired populate_variant_translations_for_score_set job; + superseded by the reverse-translation allele equivalence space. Read for existing old-model data, + never written for new score sets, dropped at read-cutover. See lib/variant_translations.py.""" + __tablename__ = "variant_translations" aa_clingen_id = Column(String, nullable=False, primary_key=True) diff --git a/src/mavedb/models/vep_allele_consequence.py b/src/mavedb/models/vep_allele_consequence.py new file mode 100644 index 000000000..8d6eaeacb --- /dev/null +++ b/src/mavedb/models/vep_allele_consequence.py @@ -0,0 +1,99 @@ +from datetime import date +from typing import TYPE_CHECKING + +from sqlalchemy import Column, Date, Enum, ForeignKey, Index, Integer, String, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime +from mavedb.models.enums.vep import VepConsequenceSource + +if TYPE_CHECKING: + from .allele import Allele + + +class VepAlleleConsequence(ValidTime, Base): + """Valid-time VEP functional-consequence result for a deduplicated :class:`Allele`. + + A row is live while ``valid_to`` is NULL; the partial unique index enforces a + single live consequence per allele, matching the gnomAD link shape (not ClinVar's multi-live shape). + + Two version axes key a stored consequence, and the job's current-release skip requires *both* to + match: + + - ``source_version`` — the Ensembl release the consequence was resolved under (e.g. ``"116"``), + version-keying the upstream result like gnomAD's ``db_version``. Always the *latest* release that + confirmed the value, not the release it first appeared. + - ``resolver_version`` — ``variant_annotation.lib.vep.RESOLVER_VERSION``, the version of *our* + resolution rule (severity ranking, transcript matching, Recoder combination). Catches a rule fix + the Ensembl release can't see; a NULL (legacy) row never matches and is re-queried once, then + filled in place. + + Supersede is **value-keyed, not version-keyed** (the one divergence from gnomAD): a re-run + confirming the *same* consequence advances both versions and ``access_date`` in place rather than + churning history; only a *changed* consequence retires the old row and inserts a successor. + ``access_date`` is a human-facing "last confirmed" stamp — it plays no part in the skip itself. + + ``functional_consequence`` is nullable to leave room for a future negative cache; the current job + writes only non-null consequences and re-queries no-result alleles each run. + + **Resolution provenance (#772).** ``functional_consequence`` is the headline term; three columns + record *how* it was reached, since VEP's cross-transcript ``most_severe`` headline routinely + describes a different overlapping isoform than the allele's own transcript: + + - ``consequence_terms`` — every term from the matched transcript entry, severity-ordered; + ``functional_consequence`` is its first element. + - ``consequence_source`` — transcript-matched, cross-transcript headline, or reference-identical. + - ``matched_transcript`` — the transcript VEP actually used, set only when ``consequence_source = + 'transcript'`` (not assumed equal to the allele's own transcript, since the match is + version-insensitive). + + Transient request failures are not stored here, they live in the annotation event stream + (``AnnotationEvent``). + """ + + __tablename__ = "vep_allele_consequences" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + functional_consequence = Column(String, nullable=True) + consequence_terms = Column(JSONB(none_as_null=True), nullable=True) + consequence_source = Column( + Enum( + VepConsequenceSource, + name="vepconsequencesource", + create_constraint=True, + length=32, + native_enum=False, + validate_strings=True, + ), + nullable=True, + ) + matched_transcript = Column(String, nullable=True) + source_version: Mapped[str] = Column(String, nullable=False) + # Nullable only for rows predating the column (treated as stale — re-queried, then filled). + resolver_version = Column(String, nullable=True) + access_date: Mapped[date] = Column(Date, nullable=False) + + allele: Mapped["Allele"] = relationship("Allele") + + __table_args__ = ( + Index( + "ix_vep_allele_consequences_allele_id", + "allele_id", + ), + # At most one live consequence per allele. A changed result supersedes (retires the old, + # inserts the new) rather than accumulating; superseded rows stay for point-in-time queries. + # Only the live row participates in this constraint. + Index( + "uq_vep_allele_consequences_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/routers/alleles.py b/src/mavedb/routers/alleles.py new file mode 100644 index 000000000..dcd90309c --- /dev/null +++ b/src/mavedb/routers/alleles.py @@ -0,0 +1,147 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Path, Query, Response +from fastapi.exceptions import HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session +from starlette.convertors import Convertor, register_url_convertor + +from mavedb import deps +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.lib.alleles import readable_score_set_ids_by_allele +from mavedb.lib.authentication import get_current_user +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.types.authentication import UserData +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.routers.shared import ( + ACCESS_CONTROL_ERROR_RESPONSES, + PUBLIC_ERROR_RESPONSES, + ROUTER_BASE_PREFIX, +) +from mavedb.view_models.allele_detail import AlleleDetail + +TAG_NAME = "Alleles" + +logger = logging.getLogger(__name__) + + +# One resource, three identifier forms — a GA4GH VRS digest (one allele), a ClinGen nucleotide id (CAID, +# the nt-canonical change) or protein id (PAID). Rather than split the path or add a query param, overload +# a single `/alleles/{identifier}` with a custom Starlette convertor (the same pattern the publication +# router uses for DOI/PubMed/…). The convertor matches any of the three forms; the handler dispatches by +# form. In the OpenAPI spec this is still just a string path parameter. +class AlleleIdentifierConverter(Convertor): + # A GA4GH IR (ga4gh:.<32 base64url>) OR a ClinGen allele id (CA…/PA… + digits). + # NOTE: Multivariant CAIDs (CA…/PA… + digits + `,` + CA…/PA… + digits) are intentionally excluded. + regex = r"(?:ga4gh:[^.]+\.[0-9A-Za-z_\-]{32})|(?:[CP]A[0-9]+)" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return str(value) + + +register_url_convertor("allele_identifier", AlleleIdentifierConverter()) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "Retrieve deduplicated alleles — identity, cross-layer equivalence class, and " + "external annotations — by VRS digest or ClinGen allele id (CAID / PAID).", +} + + +def _representative(alleles: list[Allele]) -> Allele: + """The allele whose identity fills the flat anchor fields for a ClinGen-id fetch. A CAID names one nt + change in up to two frames (genomic + coding) — prefer the coding frame, then genomic; a PAID matches + the single protein allele, which falls through to ``alleles[0]``.""" + by_level = {a.level: a for a in alleles} + return by_level.get(SequenceLevel.cdna.value) or by_level.get(SequenceLevel.genomic.value) or alleles[0] + + +@router.get( + "/alleles/{identifier:allele_identifier}", + status_code=200, + response_model=AlleleDetail, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="Fetch allele detail by VRS digest, CAID, or PAID", +) +def get_allele( + *, + response: Response, + identifier: str = Path( + description="A GA4GH VRS digest (one allele), or a ClinGen allele id: a nucleotide CAID " + "(the nt-canonical change, its genomic + coding frames) or a protein PAID.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (equivalence-class membership + VEP/gnomAD/ClinVar " + "annotations) as it stood at this instant. ISO 8601, ideally timezone-aware. The focus " + "allele's own identity is content-addressed and immutable, so it is unaffected. Defaults to " + "current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """Fetch the detail envelope for a deduplicated allele, by any of its identifiers. + + The allele-grain counterpart of ``GET /variants/{urn}``. Flat anchor identity (digest, level, HGVS, + ClinGen id, spec-pure VRS) plus the cross-layer equivalence class (each member labelled relative to + the focus) and a digest-keyed annotation map. The ``identifier`` may be: + + - a **VRS digest** (``ga4gh:VA.…``) — focuses that one allele. + - a **CAID** (``CA…``) — the nt-canonical change; The coding frame is the preferential focus, + falling back to the genomic frame if no coding frame exists. + - a **PAID** (``PA…``) — the protein change; the protein allele is focused and its nucleotide + equivalents surface as reverse-translation candidates. + + The envelope carries no score-set-level information. No scores, classifications, measurements, or + version standing. Only the allele's own identity, its cross-layer equivalence class, and public + reference annotations (VEP / gnomAD / ClinVar). + + An allele is in MaveDB only because a score set's mapping produced it, so it is served only to + callers who may read at least one such score set; otherwise the response is a 404, indistinguishable + from an unknown identifier. The equivalence class is likewise limited to readable score sets. + """ + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + if identifier.startswith("ga4gh:"): + allele = db.scalar(select(Allele).where(Allele.vrs_digest == identifier)) + readable = readable_score_set_ids_by_allele(db, user_data, [allele.id]) if allele is not None else {} + if allele is None or allele.id not in readable: + raise HTTPException(status_code=404, detail=f"allele with VRS digest '{identifier}' not found") + + return get_allele_detail(db, allele, focus_digests={identifier}, as_of=as_of, score_set_ids=readable[allele.id]) + + # A ClinGen allele id (CAID or PAID): resolve to its allele(s) — the nt-canonical change's genomic + + # coding frames for a CAID, or the single protein allele for a PAID — and focus the readable ones. + candidates = db.scalars(select(Allele).where(Allele.clingen_allele_id == identifier)).all() + readable = readable_score_set_ids_by_allele(db, user_data, [a.id for a in candidates]) + matches = [a for a in candidates if a.id in readable] + if not matches: + raise HTTPException(status_code=404, detail=f"no allele with ClinGen allele id '{identifier}' found") + + focus_digests = {a.vrs_digest for a in matches if a.vrs_digest is not None} + return get_allele_detail( + db, + _representative(matches), + focus_digests=focus_digests, + as_of=as_of, + score_set_ids=set().union(*readable.values()), + ) diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py new file mode 100644 index 000000000..a686f3ff7 --- /dev/null +++ b/src/mavedb/routers/clingen_alleles.py @@ -0,0 +1,95 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Query, Response +from sqlalchemy.orm import Session + +from mavedb import deps +from mavedb.lib.allele_measurements import get_allele_measurements +from mavedb.lib.authentication import get_current_user +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.types.authentication import UserData +from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX +from mavedb.view_models.allele_measurement import AlleleMeasurement + +TAG_NAME = "ClinGen alleles" + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/clingen-alleles", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "The ClinGen-allele-centric variant page: measurements across a variant's equivalence class.", +} + + +@router.get( + "/{clingen_allele_id}/measurements", + status_code=200, + response_model=list[AlleleMeasurement], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="List measurements for a ClinGen allele's equivalence class", +) +def get_clingen_allele_measurements( + *, + clingen_allele_id: str, + response: Response, + include_superseded: bool = Query( + default=False, + description=( + "Include measurements from superseded score-set versions. Default false — superseded " + "measurements are a deliberate power-user / citation path, never surfaced by discovery." + ), + ), + include_research_use_only: bool = Query( + default=False, + description=( + "Let a call from a research-use-only calibration stand in when the score set has no readable " + "calibration that is not research use only. Such calls set classificationIsResearchUseOnly and are " + "not clinical evidence. Default false; ordering never uses them." + ), + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the equivalence class (which mapping records / allele links are live) as it " + "stood at this instant. ISO 8601, ideally timezone-aware. Scores and classifications are " + "as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """List every measurement whose cross-layer equivalence class touches this ClinGen allele (a ``CA`` or + ``PA``) — the direct measurements assayed at this change plus the reverse-translation-related ones, + each labeled by its assayed level and relationship. This is the ClinGen-allele-centric variant page's + entrypoint. A private score set's measurement is never included; its inline classification is withheld + where the calibration is unreadable while the measurement still shows. + """ + save_to_logging_context( + { + "requested_resource": clingen_allele_id, + "as_of": as_of, + "include_superseded": include_superseded, + "include_research_use_only": include_research_use_only, + } + ) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + return get_allele_measurements( + db, + clingen_allele_id, + user_data=user_data, + include_superseded=include_superseded, + include_research_use_only=include_research_use_only, + as_of=as_of, + ) diff --git a/src/mavedb/routers/collections.py b/src/mavedb/routers/collections.py index 8475322d7..051d5bb3a 100644 --- a/src/mavedb/routers/collections.py +++ b/src/mavedb/routers/collections.py @@ -241,11 +241,16 @@ async def create_collection( score_set_associations = [] for position, score_set_urn in enumerate(item_create.score_set_urns or []): score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one() + # A record the caller may not read is reported as missing, so membership can't probe for it. + if not has_permission(user_data, score_set, Action.READ).permitted: + raise NoResultFound() score_set_associations.append(CollectionScoreSetAssociation(score_set=score_set, position=position)) experiment_associations = [] for position, experiment_urn in enumerate(item_create.experiment_urns or []): experiment = db.scalars(select(Experiment).where(Experiment.urn == experiment_urn)).one() + if not has_permission(user_data, experiment, Action.READ).permitted: + raise NoResultFound() experiment_associations.append(CollectionExperimentAssociation(experiment=experiment, position=position)) except NoResultFound as e: @@ -345,6 +350,13 @@ async def update_collection( # Build new ordered associations for position, score_set_urn in enumerate(item_update.score_set_urns): score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one() + # Only new members are checked, so existing members the caller can't see survive a reorder. + # A new member the caller may not read is reported as missing. + if ( + score_set_urn not in current_urns + and not has_permission(user_data, score_set, Action.READ).permitted + ): + raise NoResultFound() item.score_set_associations.append( CollectionScoreSetAssociation(score_set=score_set, position=position) ) @@ -372,6 +384,11 @@ async def update_collection( # Build new ordered associations for position, experiment_urn in enumerate(item_update.experiment_urns): experiment = db.scalars(select(Experiment).where(Experiment.urn == experiment_urn)).one() + if ( + experiment_urn not in current_urns + and not has_permission(user_data, experiment, Action.READ).permitted + ): + raise NoResultFound() item.experiment_associations.append( CollectionExperimentAssociation(experiment=experiment, position=position) ) @@ -450,7 +467,8 @@ async def add_score_set_to_collection( raise HTTPException(status_code=404, detail=f"collection with URN '{collection_urn}' not found") score_set = db.execute(select(ScoreSet).where(ScoreSet.urn == body.score_set_urn)).scalars().one_or_none() - if not score_set: + # A score set the caller may not read is reported as missing, so this route can't probe for one. + if not score_set or not has_permission(user_data, score_set, Action.READ).permitted: logger.info( msg="Failed to add score set to collection; The requested score set does not exist.", extra=logging_context(), @@ -608,7 +626,8 @@ async def add_experiment_to_collection( raise HTTPException(status_code=404, detail=f"collection with URN '{collection_urn}' not found") experiment = db.execute(select(Experiment).where(Experiment.urn == body.experiment_urn)).scalars().one_or_none() - if not experiment: + # An experiment the caller may not read is reported as missing, so this route can't probe for one. + if not experiment or not has_permission(user_data, experiment, Action.READ).permitted: logger.info( msg="Failed to add experiment to collection; The requested experiment does not exist.", extra=logging_context(), diff --git a/src/mavedb/routers/diseases.py b/src/mavedb/routers/diseases.py new file mode 100644 index 000000000..f0f2f9daa --- /dev/null +++ b/src/mavedb/routers/diseases.py @@ -0,0 +1,43 @@ +from typing import Any + +from fastapi import APIRouter, Query +from ga4gh.core.models import MappableConcept + +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.logging.logged_route import LoggedRoute +from mavedb.lib.mondo import mondo_suggestion_to_mappable_concept +from mavedb.lib.mondo_ols import DEFAULT_SEARCH_LIMIT, search_mondo +from mavedb.routers.shared import GATEWAY_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX + +TAG_NAME = "Diseases" + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/diseases", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "Search disease terms from the Monarch Disease Ontology (MONDO).", +} + + +@router.get( + "/search", + status_code=200, + response_model=list[MappableConcept], + summary="Search MONDO disease terms", + responses={**GATEWAY_ERROR_RESPONSES}, +) +async def search_diseases( + *, + q: str = Query(..., description="Free-text query for a MONDO disease term."), + limit: int = Query(DEFAULT_SEARCH_LIMIT, ge=1, le=100, description="Maximum number of results."), +) -> Any: + """Typeahead search for MONDO disease terms, returned as GA4GH ``MappableConcept`` suggestions.""" + save_to_logging_context({"query": q, "limit": limit}) + + suggestions = await search_mondo(q, limit) + return [mondo_suggestion_to_mappable_concept(suggestion) for suggestion in suggestions] diff --git a/src/mavedb/routers/experiments.py b/src/mavedb/routers/experiments.py index b78ca27d0..a58d127b9 100644 --- a/src/mavedb/routers/experiments.py +++ b/src/mavedb/routers/experiments.py @@ -13,6 +13,7 @@ from mavedb.lib.authorization import get_principal, require_current_user, require_current_user_with_email from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import NonexistentOrcidUserError +from mavedb.lib.collections import readable_official_collections from mavedb.lib.experiments import enrich_experiment_with_num_score_sets from mavedb.lib.experiments import search_experiments as _search_experiments from mavedb.lib.identifiers import ( @@ -26,7 +27,7 @@ from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import find_superseded_score_set_tail +from mavedb.lib.score_sets import find_superseded_score_set_tail, readable_score_set_urns from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.validation.keywords import validate_keyword_list @@ -79,7 +80,7 @@ def list_experiments( editable: Optional[bool] = None, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), -) -> list[Experiment]: +) -> list[experiment.Experiment]: """ List all experiments viewable by the current user. """ @@ -99,7 +100,11 @@ def list_experiments( ) items = query.order_by(Experiment.urn).all() - return [item for item in items if has_permission(user_data, item, Action.READ).permitted] + return [ + enrich_experiment_with_num_score_sets(item, user_data) + for item in items + if has_permission(user_data, item, Action.READ).permitted + ] @router.post( @@ -237,15 +242,31 @@ def get_experiment_score_sets( for fs in filtered_score_sets: enriched_experiment = enrich_experiment_with_num_score_sets(fs.experiment, user_data) visible_calibration_ids = {calibration.id for calibration in viewer.visible(fs.score_calibrations)} + # A superseded score set reaches this list *because* its successor failed the caller's READ check + # (see find_superseded_score_set_tail), so serializing that successor's urn and title would name + # the very score set the caller was found not to be entitled to. + # TODO(#808): this duplicates score_sets._score_set_response; the two have already drifted once. + superseding_is_visible = fs.superseding_score_set is not None and ( + has_permission(user_data, fs.superseding_score_set, Action.READ).permitted + ) + readable_meta_analysis_urns = readable_score_set_urns( + user_data, + fs.meta_analyzed_by_score_sets, # type: ignore[attr-defined] # backref of meta_analyzes_score_sets + ) validated_item = score_set.ScoreSet.model_validate(fs) response_item = validated_item.copy( update={ "experiment": enriched_experiment, + "meta_analyzed_by_score_set_urns": [ + urn for urn in validated_item.meta_analyzed_by_score_set_urns if urn in readable_meta_analysis_urns + ], + "official_collections": readable_official_collections(user_data, fs.official_collections), "score_calibrations": [ calibration for calibration in (validated_item.score_calibrations or []) if calibration.id in visible_calibration_ids ], + "superseding_score_set": validated_item.superseding_score_set if superseding_is_visible else None, } ) enriched_score_sets.append(response_item) diff --git a/src/mavedb/routers/hgvs.py b/src/mavedb/routers/hgvs.py index c2352c81d..ac6277300 100644 --- a/src/mavedb/routers/hgvs.py +++ b/src/mavedb/routers/hgvs.py @@ -5,10 +5,11 @@ from cdot.hgvs.dataproviders import RESTDataProvider from fastapi import APIRouter, Depends, HTTPException from hgvs import parser, validator -from hgvs.exceptions import HGVSDataNotAvailableError, HGVSInvalidVariantError +from hgvs.exceptions import HGVSDataNotAvailableError, HGVSError from mavedb.deps import hgvs_data_provider -from mavedb.routers.shared import BASE_400_RESPONSE, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX +from mavedb.routers.shared import PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, VALIDATION_ERROR_RESPONSES +from mavedb.view_models.hgvs import HgvsValidationRequest TAG_NAME = "Transcripts" @@ -44,22 +45,23 @@ def hgvs_fetch(accession: str, hdp: RESTDataProvider = Depends(hgvs_data_provide "/validate", status_code=200, response_model=bool, - responses={**BASE_400_RESPONSE}, + responses={**VALIDATION_ERROR_RESPONSES}, summary="Validate a provided variant", ) -def hgvs_validate(variant: dict[str, str], hdp: RESTDataProvider = Depends(hgvs_data_provider)) -> bool: +def hgvs_validate(request: HgvsValidationRequest, hdp: RESTDataProvider = Depends(hgvs_data_provider)) -> bool: """ Validate the provided HGVS variant string. + + Parsing and validation failures both stem from caller-supplied input, so any ``HGVSError`` — a syntactic + parse failure, an inconsistent variant, an unknown accession — is surfaced as a 400 rather than escaping + to the catch-all 500 handler. """ hp = parser.Parser() - variant_hgvs = hp.parse(variant["variant"]) - try: - valid = validator.Validator(hdp=hdp).validate(variant_hgvs, strict=False) - except HGVSInvalidVariantError as e: + variant_hgvs = hp.parse(request.variant) + return validator.Validator(hdp=hdp).validate(variant_hgvs, strict=False) + except HGVSError as e: raise HTTPException(400, str(e)) - else: - return valid @router.get("/assemblies", status_code=200, response_model=list[str], summary="List stored assemblies") diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py index f6bfb7f97..244e490ff 100644 --- a/src/mavedb/routers/mapped_variant.py +++ b/src/mavedb/routers/mapped_variant.py @@ -1,232 +1,102 @@ import logging -from typing import Annotated, Any, Optional +from typing import Annotated +from urllib.parse import quote -from fastapi import APIRouter, Depends, Path -from fastapi.exceptions import HTTPException +from fastapi import APIRouter, Path, Request +from fastapi.responses import RedirectResponse from ga4gh.core.identifiers import GA4GH_IR_REGEXP -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session -from mavedb import deps -from mavedb.lib.annotation.annotate import ( - variant_functional_impact_statement, - variant_pathogenicity_statement, - variant_study_result, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.authorization import get_current_user, get_principal +from mavedb.lib.deprecation import MAPPED_VARIANT_SUNSET, deprecation_headers, record_deprecated_usage from mavedb.lib.logging import LoggedRoute -from mavedb.lib.permissions.principal import Principal -from mavedb.lib.logging.context import ( - logging_context, - save_to_logging_context, -) -from mavedb.lib.permissions import Action, assert_permission, has_permission -from mavedb.lib.types.authentication import UserData -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX -from mavedb.view_models import mapped_variant +from mavedb.routers.shared import ROUTER_BASE_PREFIX -TAG_NAME = "Mapped Variants" +TAG_NAME = "Variants" logger = logging.getLogger(__name__) - -async def fetch_mapped_variant_by_variant_urn(db: Session, user: Optional[UserData], urn: str) -> MappedVariant: - """ - We may combine this function back to show_mapped_variant if none of any new function call it. - Fetch one mapped variant by variant URN. - - :param db: An active database session. - :param urn: The variant URN. - :return: The mapped variant. - - :raises HTTPException: If the mapped variant is not found or if multiple variants are found. - """ - try: - item = ( - db.execute(select(MappedVariant).join(Variant).where(Variant.urn == urn, MappedVariant.current.is_(True))) - .scalars() - .one_or_none() - ) - except MultipleResultsFound: - logger.info( - msg="Could not fetch the requested mapped variant; Multiple such variants exist.", extra=logging_context() - ) - raise HTTPException(status_code=500, detail=f"Multiple variants with URN {urn} were found.") - - if not item: - logger.info( - msg="Could not fetch the requested mapped variant; No such mapped variants exist.", extra=logging_context() - ) - - raise HTTPException(status_code=404, detail=f"Mapped variant with URN {urn} not found") - - # Base mapped variant read permission on the score set in which it is contained. - assert_permission(user, item.variant.score_set, Action.READ) - return item - - router = APIRouter( prefix=f"{ROUTER_BASE_PREFIX}/mapped-variants", tags=[TAG_NAME], - responses={**PUBLIC_ERROR_RESPONSES}, route_class=LoggedRoute, ) -metadata = { - "name": TAG_NAME, - "description": "Retrieve mapped variants and their associated variant annotations.", -} + +def _redirect_to_successor(request: Request, target: str) -> RedirectResponse: + """Redirect to the successor, tagging the response with RFC 8594 deprecation headers. + + The redirect status already moves a well-behaved client, but the headers announce the deprecation to + tooling that follows the 301 silently: ``Deprecation``, a ``Link`` naming this exact successor, and a + ``Warning`` a human sees in logs. The successor here is a genuine 1:1 replacement, so it is safe to name. + """ + # The ASGI scope rather than request.url: a mapped-variant URN's '#' opens a fragment when Starlette + # rebuilds request.url by re-parsing the decoded path, which silently drops the query string too. + path = request.scope["path"] + query = request.scope.get("query_string", b"").decode("ascii") + url = f"{target}?{query}" if query else target + record_deprecated_usage(path, successor=target) + headers = deprecation_headers( + successor=target, + warning=f"This resource has moved permanently to {target}; update your integration.", + sunset=MAPPED_VARIANT_SUNSET, + ) + return RedirectResponse(url=url, status_code=301, headers=headers) @router.get( "/{urn}", - status_code=200, - response_model=mapped_variant.MappedVariantWithMappingDetails, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variant by URN", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}", ) -async def show_mapped_variant( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> Any: - """ - Fetch a single mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - return await fetch_mapped_variant_by_variant_urn(db, user, urn) +def redirect_mapped_variant(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}`` instead.""" + return _redirect_to_successor(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}") @router.get( "/{urn}/va/study-result", - status_code=200, - response_model=ExperimentalVariantFunctionalImpactStudyResult, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec StudyResult from a mapped variant", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/study-result", ) -async def show_mapped_variant_study_result( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> ExperimentalVariantFunctionalImpactStudyResult: - """ - Construct a single VA-Spec StudyResult from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) +def redirect_mapped_variant_study_result(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/study-result`` instead.""" + return _redirect_to_successor(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/study-result") - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - try: - return variant_study_result(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg=f"Could not construct a study result for mapped variant {urn}: {e}", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No study result exists for mapped variant {urn}: {e}") - - -# TODO#416: For now, this route supports only one statement per mapped variant. Eventually, we should support the possibility of multiple statements. @router.get( "/{urn}/va/functional-statement", - status_code=200, - response_model=Statement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec Statement from a mapped variant", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/functional-statement", ) -async def show_mapped_variant_functional_impact_statement( - *, - urn: str, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), - principal: Principal = Depends(get_principal), -) -> Statement: - """ - Construct a single VA-Spec Statement from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - functional_impact = variant_functional_impact_statement(mapped_variant, principal=principal) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, detail=f"No functional impact statement exists for mapped variant {urn}: {e}" - ) - - if not functional_impact: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant. Variant does not have sufficient evidence to evaluate its functional impact.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No functional impact statement exists for mapped variant {urn}. Variant does not have sufficient evidence to evaluate its functional impact.", - ) - - return functional_impact +def redirect_mapped_variant_functional_impact_statement(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/functional-statement`` instead.""" + return _redirect_to_successor( + request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/functional-statement" + ) -# TODO#416: For now, this route supports only one evidence line per mapped variant. Eventually, we should support the possibility of multiple evidence lines. @router.get( "/{urn}/va/pathogenicity-statement", - status_code=200, - response_model=VariantPathogenicityStatement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec EvidenceLine from a mapped variant", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/pathogenicity-statement", ) -async def show_mapped_variant_acmg_evidence_line( - *, - urn: str, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), - principal: Principal = Depends(get_principal), -) -> VariantPathogenicityStatement: - """ - Construct a list of VA-Spec EvidenceLine(s) from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - pathogenicity_statement = variant_pathogenicity_statement(mapped_variant, principal=principal) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for mapped variant {urn}: {e}") - - if not pathogenicity_statement: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; Variant does not have sufficient evidence to evaluate its pathogenicity.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No pathogenicity statement exists for mapped variant {urn}; Variant does not have sufficient evidence to evaluate its pathogenicity.", - ) - - return pathogenicity_statement +def redirect_mapped_variant_acmg_evidence_line(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/pathogenicity-statement`` instead.""" + return _redirect_to_successor( + request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/pathogenicity-statement" + ) @router.get( "/vrs/{identifier}", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variants by VRS identifier", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/vrs/{identifier}", ) -async def show_mapped_variants_by_identifier( +def redirect_mapped_variants_by_identifier( *, identifier: Annotated[ str, @@ -236,36 +106,12 @@ async def show_mapped_variants_by_identifier( regex=GA4GH_IR_REGEXP, ), ], - only_current: bool = True, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Fetch a single mapped variant by GA4GH identifier. - """ - query = select(MappedVariant).where( - or_(MappedVariant.pre_mapped["id"].astext == identifier, MappedVariant.post_mapped["id"].astext == identifier) - ) + request: Request, +) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/vrs/{identifier}`` instead. - if only_current: - query = query.where(MappedVariant.current.is_(True)) - - items = db.scalars(query).all() - - permitted_items = [item for item in items if has_permission(user, item.variant.score_set, Action.READ).permitted] - if not permitted_items: - raise HTTPException(status_code=404, detail=f"No mapped variants with identifier {identifier} were found") - - return permitted_items - - -# for testing only -# @router.post("/map/{urn}", status_code=200, responses={404: {}, 500: {}}) -# async def map_score_set(*, urn: str, worker: ArqRedis = Depends(deps.get_worker)) -> Any: -# await worker.lpush(MAPPING_QUEUE_NAME, urn) # type: ignore -# await worker.enqueue_job( -# "variant_mapper_manager", -# None, -# None, -# None -# ) + Note that the replacement's ``only_current`` boolean query parameter has been superseded by + ``as_of``; a caller relying on ``only_current=false`` should switch to passing an explicit + ``as_of`` timestamp rather than expecting it to carry over through this redirect. + """ + return _redirect_to_successor(request, f"{ROUTER_BASE_PREFIX}/variants/vrs/{quote(identifier, safe='')}") diff --git a/src/mavedb/routers/permissions.py b/src/mavedb/routers/permissions.py index 39833ec77..cd6349f66 100644 --- a/src/mavedb/routers/permissions.py +++ b/src/mavedb/routers/permissions.py @@ -76,9 +76,9 @@ async def check_permission( elif model_name == ModelName.score_calibration: item = db.query(ScoreCalibration).filter(ScoreCalibration.urn == urn).one_or_none() - if item: - permission = has_permission(user_data, item, action).permitted - return permission - else: - logger.debug(msg="The requested resources does not exist.", extra=logging_context()) - raise HTTPException(status_code=404, detail=f"{model_name.value} with URN '{urn}' not found") + # A resource the caller may not read is reported as missing, so this route can't confirm it exists. + if item and has_permission(user_data, item, Action.READ).permitted: + return has_permission(user_data, item, action).permitted + + logger.debug(msg="The requested resource does not exist or is not readable.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"{model_name.value} with URN '{urn}' not found") diff --git a/src/mavedb/routers/refget.py b/src/mavedb/routers/refget.py index 979c63d13..c19e3bbc1 100644 --- a/src/mavedb/routers/refget.py +++ b/src/mavedb/routers/refget.py @@ -33,6 +33,10 @@ logger = logging.getLogger(__name__) +# SeqRepo records no sequence topology, so the human mitochondrial references are the only sequences treated as +# circular. Refget needs the distinction: start > end is 416 on a linear sequence and 501 on a circular one. +CIRCULAR_SEQUENCE_ACCESSIONS = frozenset({"NC_012920.1", "NC_001807.4"}) + router = APIRouter( prefix=f"{ROUTER_BASE_PREFIX}/refget", tags=[TAG_NAME], @@ -158,19 +162,14 @@ def get_sequence( logger.error(msg="Invalid range header format", extra=logging_context()) raise HTTPException(status_code=400, detail="Invalid range header format") + # Rejects "bytes=5-4"-style headers (RFC 9110 S14.1.1): after conversion to a half-open + # bound, start == end here would otherwise silently pass as a valid empty range. start, end = int(m.group(1)), int(m.group(2)) + 1 + if start >= end: + logger.error(msg="Invalid range header format", extra=logging_context()) + raise HTTPException(status_code=400, detail="Invalid range header format") save_to_logging_context({"requested_refget_start": start, "requested_refget_end": end}) - if start is not None and end is not None: - if start > end: - logger.error( - msg="Invalid coordinates: start is greater than end and circular chromosomes are not supported", - extra=logging_context(), - ) - raise HTTPException( - status_code=501, - detail="Invalid coordinates: start is greater than end and circular chromosomes are not supported", - ) seq_ids = get_sequence_ids(sr, alias) if not seq_ids: @@ -183,37 +182,74 @@ def get_sequence( ) seq_id = seq_ids[0] - seqinfo = sr.sequences.fetch_seqinfo(seq_id) + seq_len = sr.sequences.fetch_seqinfo(seq_id)["len"] + + # Resolve to concrete half-open bounds up front so validation, Content-Length, and the streamed + # body all agree, even when only one of start/end is supplied. + seq_start = start if start is not None else 0 + seq_end = end if end is not None else seq_len + + # Refget spec: start > end asks for a circular sub-sequence across the origin. That's unsatisfiable on a linear + # sequence, and not implemented here for a circular one. + if start is not None and end is not None and start > end: + if not CIRCULAR_SEQUENCE_ACCESSIONS.isdisjoint(a["alias"] for a in sr.aliases.find_aliases(seq_id=seq_id)): + logger.error(msg="Circular sub-sequences are not supported", extra=logging_context()) + raise HTTPException( + status_code=501, + detail="Invalid coordinates: start is greater than end and circular chromosomes are not supported", + ) + raise HTTPException( + status_code=416, + detail="Invalid coordinates: start is greater than end on a linear sequence", + headers={"Content-Range": f"bytes */{seq_len}"}, + ) + # Refget spec: only a start "larger than" the sequence length is invalid (strict inequality). if start is not None and end is not None: - if start >= seqinfo["len"]: + if seq_start > seq_len: raise HTTPException( status_code=416, detail="Invalid coordinates: start > sequence length", - headers={"Content-Range": f"bytes */{seqinfo['len']}"}, + headers={"Content-Range": f"bytes */{seq_len}"}, ) - if end > seqinfo["len"]: + if seq_end > seq_len: raise HTTPException( status_code=416, detail="Invalid coordinates: end > sequence length", - headers={"Content-Range": f"bytes */{seqinfo['len']}"}, + headers={"Content-Range": f"bytes */{seq_len}"}, ) - if not (0 <= start <= end <= seqinfo["len"]): + if not (0 <= seq_start <= seq_end <= seq_len): raise HTTPException( status_code=416, detail="Invalid coordinates: must obey 0 <= start <= end <= sequence_length", - headers={"Content-Range": f"bytes */{seqinfo['len']}"}, + headers={"Content-Range": f"bytes */{seq_len}"}, + ) + # The refget spec only requires rejecting a start beyond the sequence length; there's no + # equivalent rule for a lone end (see below). + elif start is not None: + if not 0 <= seq_start <= seq_len: + raise HTTPException( + status_code=400, + detail=f"Invalid coordinates: start must satisfy 0 <= start <= {seq_len}", ) + # Clamp the end coordinate to the valid range [0, seq_len]. + elif end is not None: + seq_end = max(0, min(seq_end, seq_len)) - headers = {"Content-Length": str(seqinfo["len"])} - if start is not None and end is not None and range_header: + # Content-Length must match bytes actually streamed. Overstating it aborts the response mid-stream + # once the ASGI server has already committed the status line. + headers = {"Content-Length": str(seq_end - seq_start)} + if range_header: status = 206 - headers["Content-Range"] = f"bytes {start}-{end - 1}/{seqinfo['len']}" + headers["Content-Range"] = f"bytes {seq_start}-{seq_end - 1}/{seq_len}" headers["Accept-Ranges"] = "bytes" else: status = 200 headers["Accept-Ranges"] = "none" return StreamingResponse( - sequence_generator(sr, seq_ids[0], start, end), media_type="text/plain", status_code=status, headers=headers + sequence_generator(sr, seq_id, seq_start, seq_end), + media_type="text/plain", + status_code=status, + headers=headers, ) diff --git a/src/mavedb/routers/score_calibrations.py b/src/mavedb/routers/score_calibrations.py index 1df20b5ee..484575381 100644 --- a/src/mavedb/routers/score_calibrations.py +++ b/src/mavedb/routers/score_calibrations.py @@ -25,8 +25,17 @@ ) from mavedb.lib.score_sets import csv_data_to_df from mavedb.lib.types.authentication import UserData -from mavedb.lib.validation.constants.general import calibration_class_column_name, calibration_variant_column_name -from mavedb.lib.validation.dataframe.calibration import validate_and_standardize_calibration_classes_dataframe +from mavedb.lib.validation.constants.general import ( + calibration_class_column_name, + calibration_control_status_column_name, + calibration_variant_column_name, + hgvs_nt_column, + hgvs_pro_column, +) +from mavedb.lib.validation.dataframe.calibration import ( + validate_and_standardize_calibration_classes_dataframe, + validate_and_standardize_calibration_controls_dataframe, +) from mavedb.lib.validation.exceptions import ValidationError from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -87,7 +96,7 @@ def list_my_calibrations( @router.get( "/{urn}", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) def get_score_calibration( @@ -157,7 +166,7 @@ async def get_score_calibrations_for_score_set( @router.get( "/score-set/{score_set_urn}/primary", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) async def get_primary_score_calibrations_for_score_set( @@ -211,7 +220,7 @@ async def get_primary_score_calibrations_for_score_set( @router.post( "/", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}, 422: {"description": "Validation Error"}}, openapi_extra={ "requestBody": { @@ -233,6 +242,11 @@ async def get_primary_score_calibrations_for_score_set( "format": "binary", "description": "CSV file containing variant classifications", }, + "controls_file": { + "type": "string", + "format": "binary", + "description": "CSV file containing calibration controls", + }, }, } }, @@ -248,6 +262,10 @@ async def create_score_calibration_route( None, description=f"CSV file containing variant classifications. This file must contain two columns: '{calibration_variant_column_name}' and '{calibration_class_column_name}'.", ), + controls_file: Optional[UploadFile] = File( + None, + description=f"CSV file containing calibration controls. This file must contain a variant column (one of '{calibration_variant_column_name}', '{hgvs_nt_column}', '{hgvs_pro_column}') and a '{calibration_control_status_column_name}' column.", + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), ) -> ScoreCalibration: @@ -281,6 +299,7 @@ async def create_score_calibration_route( **Form Fields**: - `calibration_json` (string, required): JSON string containing the calibration data - `classes_file` (file, optional): CSV file containing variant classifications + - `controls_file` (file, optional): CSV file containing calibration controls **Example**: ```bash @@ -296,11 +315,17 @@ async def create_score_calibration_route( - If uploading a classes_file, it must be a valid CSV with variant classification data - User must have ADD_CALIBRATION permission on the score set (any authenticated user for published sets; contributors/owners/admins for private sets) - + ## File Upload Details The `classes_file` parameter accepts CSV files containing variant classification data. The file should have appropriate headers and contain columns for variant urns and class names. + The `controls_file` parameter accepts a CSV of calibration controls with a variant column + (one of `variant_urn`, `hgvs_nt`, `hgvs_pro`) and a `clinical_status` column (`pathogenic` or + `benign`, case-insensitive). Controls may be supplied either via this file or inline in + `calibration_json`, but not both. Controls are accepted only with `controls_not_phi: true`, the + submitter's affirmation that they contain no protected health information (PHI). + ## Response Returns the created score calibration with its generated URN and associated score set information. """ @@ -347,9 +372,38 @@ async def create_score_calibration_route( detail=[{"loc": [e.custom_loc or "classesFile"], "msg": str(e), "type": "value_error"}], ) - created_calibration = await create_score_calibration_in_score_set( - db, calibration, user_data.user, variant_classes if classes_file else None - ) + # Controls may be supplied inline in the JSON payload or as a CSV, but not both. + if calibration.controls is not None and controls_file: + raise HTTPException( + status_code=422, + detail="Provide calibration controls either inline or via controls_file, not both.", + ) + + if controls_file: + try: + controls_df = csv_data_to_df(controls_file.file, induce_hgvs_cols=False) + except UnicodeDecodeError as e: + raise HTTPException( + status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values." + ) + + try: + calibration.controls = validate_and_standardize_calibration_controls_dataframe(db, score_set, controls_df) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": [e.custom_loc or "controlsFile"], "msg": str(e), "type": "value_error"}], + ) + + try: + created_calibration = await create_score_calibration_in_score_set( + db, calibration, user_data.user, variant_classes if classes_file else None + ) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": e.custom_loc or ["body"], "msg": str(e), "type": "value_error"}], + ) db.commit() db.refresh(created_calibration) @@ -359,7 +413,7 @@ async def create_score_calibration_route( @router.put( "/{urn}", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}, 422: {"description": "Validation Error"}}, openapi_extra={ "requestBody": { @@ -381,6 +435,11 @@ async def create_score_calibration_route( "format": "binary", "description": "CSV file containing updated variant classifications", }, + "controls_file": { + "type": "string", + "format": "binary", + "description": "CSV file containing updated calibration controls (replaces existing)", + }, }, } }, @@ -397,6 +456,10 @@ async def modify_score_calibration_route( None, description=f"CSV file containing variant classifications. This file must contain two columns: '{calibration_variant_column_name}' and '{calibration_class_column_name}'.", ), + controls_file: Optional[UploadFile] = File( + None, + description=f"CSV file containing calibration controls. This file must contain a variant column (one of '{calibration_variant_column_name}', '{hgvs_nt_column}', '{hgvs_pro_column}') and a '{calibration_control_status_column_name}' column. Replaces existing controls.", + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), ) -> ScoreCalibration: @@ -430,6 +493,7 @@ async def modify_score_calibration_route( **Form Fields**: - `calibration_json` (string, required): JSON string containing the calibration update data - `classes_file` (file, optional): CSV file containing updated variant classifications + - `controls_file` (file, optional): CSV file containing calibration controls (replaces existing) **Example**: ```bash @@ -451,6 +515,16 @@ async def modify_score_calibration_route( The file should have appropriate headers and follow the expected format for variant classifications within the associated score set. + The `controls_file` parameter accepts a CSV of calibration controls with a variant column + (one of `variant_urn`, `hgvs_nt`, `hgvs_pro`) and a `clinical_status` column (`pathogenic` or + `benign`, case-insensitive). If provided, it replaces the calibration's existing controls. + Controls may be supplied either via this file or inline in `calibration_json`, but not both; + omitting both leaves existing controls unchanged. + + Controls are stored only while `controls_not_phi` is true. New controls must be submitted with + `controls_not_phi: true` in the same request; setting it to false or null without new controls + deletes the existing controls. + ## Response Returns the updated score calibration with all modifications applied and any new classification data from the uploaded file. @@ -515,9 +589,41 @@ async def modify_score_calibration_route( detail=[{"loc": [e.custom_loc or "classesFile"], "msg": str(e), "type": "value_error"}], ) - updated_calibration = await modify_score_calibration( - db, item, calibration_update, user_data.user, variant_classes if classes_file else None - ) + # Controls may be supplied inline in the JSON payload or as a CSV, but not both. A controls_file + # replaces existing controls; omitting both leaves them unchanged (see modify replace semantics). + if calibration_update.controls is not None and controls_file: + raise HTTPException( + status_code=422, + detail="Provide calibration controls either inline or via controls_file, not both.", + ) + + if controls_file: + try: + controls_df = csv_data_to_df(controls_file.file, induce_hgvs_cols=False) + except UnicodeDecodeError as e: + raise HTTPException( + status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values." + ) + + try: + calibration_update.controls = validate_and_standardize_calibration_controls_dataframe( + db, score_set, controls_df + ) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": [e.custom_loc or "controlsFile"], "msg": str(e), "type": "value_error"}], + ) + + try: + updated_calibration = await modify_score_calibration( + db, item, calibration_update, user_data.user, variant_classes if classes_file else None + ) + except ValidationError as e: + raise HTTPException( + status_code=422, + detail=[{"loc": e.custom_loc or ["body"], "msg": str(e), "type": "value_error"}], + ) db.commit() db.refresh(updated_calibration) @@ -562,7 +668,7 @@ async def delete_score_calibration_route( @router.post( "/{urn}/promote-to-primary", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) async def promote_score_calibration_to_primary_route( @@ -578,7 +684,11 @@ async def promote_score_calibration_to_primary_route( Promote a score calibration to be the primary calibration for its associated score set. """ save_to_logging_context( - {"requested_resource": urn, "resource_property": "primary", "demote_existing_primary": demote_existing_primary} + { + "requested_resource": urn, + "resource_property": "primary", + "demote_existing_primary": demote_existing_primary, + } ) item = ( @@ -591,6 +701,10 @@ async def promote_score_calibration_to_primary_route( logger.debug("The requested score calibration does not exist", extra=logging_context()) raise HTTPException(status_code=404, detail="The requested score calibration does not exist") + # CHANGE_RANK admits score set contributors to community calibrations they may not READ. Only public + # calibrations can be ranked, so asserting READ first refuses nothing legitimate and keeps this route + # from returning, or describing, a calibration the caller cannot see. + assert_permission(user_data, item, Action.READ) assert_permission(user_data, item, Action.CHANGE_RANK) if item.primary: @@ -631,7 +745,7 @@ async def promote_score_calibration_to_primary_route( @router.post( "/{urn}/demote-from-primary", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) def demote_score_calibration_from_primary_route( @@ -655,6 +769,10 @@ def demote_score_calibration_from_primary_route( logger.debug("The requested score calibration does not exist", extra=logging_context()) raise HTTPException(status_code=404, detail="The requested score calibration does not exist") + # CHANGE_RANK admits score set contributors to community calibrations they may not READ. Only public + # calibrations can be ranked, so asserting READ first refuses nothing legitimate and keeps this route + # from returning, or describing, a calibration the caller cannot see. + assert_permission(user_data, item, Action.READ) assert_permission(user_data, item, Action.CHANGE_RANK) if not item.primary: @@ -670,7 +788,7 @@ def demote_score_calibration_from_primary_route( @router.post( "/{urn}/publish", - response_model=score_calibration.ScoreCalibrationWithScoreSetUrn, + response_model=score_calibration.ScoreCalibrationDetailWithScoreSetUrn, responses={404: {}}, ) def publish_score_calibration_route( @@ -739,7 +857,11 @@ def get_functional_classification_variants( a `variant_count` summary for performance. """ save_to_logging_context( - {"requested_resource": urn, "requested_classification": classification_id, "resource_property": "variants"} + { + "requested_resource": urn, + "requested_classification": classification_id, + "resource_property": "variants", + } ) calibration = ( diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 1bab6230c..425b1f49c 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -10,7 +10,7 @@ import pandas as pd import requests from arq import ArqRedis -from fastapi import APIRouter, Depends, File, Query, Request, UploadFile +from fastapi import APIRouter, Depends, File, Query, Request, Response, UploadFile from fastapi.encoders import jsonable_encoder from fastapi.exceptions import HTTPException, RequestValidationError from fastapi.responses import StreamingResponse @@ -19,15 +19,19 @@ from pydantic import ValidationError from sqlalchemy import or_, select from sqlalchemy.exc import IntegrityError, MultipleResultsFound -from sqlalchemy.orm import Session, contains_eager +from sqlalchemy.orm import Session from mavedb import deps from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, s3_client +from mavedb.db.timeouts import allow_long_statements +from mavedb.lib.alleles import with_chunk_inputs from mavedb.lib.annotation.annotate import ( variant_functional_impact_statement, variant_pathogenicity_statement, variant_study_result, ) +from mavedb.lib.annotation.conformance import serialize_annotation +from mavedb.lib.annotation.context import AnnotationContextInputs, load_annotation_context_inputs from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.authorization import ( get_current_user, @@ -35,7 +39,10 @@ require_current_user, require_current_user_with_email, ) +from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns +from mavedb.lib.collections import readable_official_collections from mavedb.lib.contributors import find_or_create_contributor +from mavedb.lib.csv.build_limiter import csv_build_slot, rows_to_build from mavedb.lib.csv.columns import variants_to_csv_rows from mavedb.lib.csv.deprecated_params import ( DROP_NA_COLUMNS_DESCRIPTION, @@ -45,29 +52,29 @@ ) from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv +from mavedb.lib.deprecation import MAPPED_VARIANT_SUNSET, deprecation_headers, record_deprecated_usage from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets +from mavedb.lib.gnomad import get_gnomad_variants_with_variant_urns from mavedb.lib.identifiers import ( create_external_gene_identifier_offset, find_or_create_doi_identifier, find_or_create_publication_identifier, ) from mavedb.lib.logging import LoggedRoute -from mavedb.lib.logging.context import ( - correlation_id_for_context, - logging_context, - save_to_logging_context, -) +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration +from mavedb.lib.score_set_variants import get_lean_score_set_variants from mavedb.lib.score_sets import ( csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, - get_current_mapped_variants_for_annotation, + get_annotatable_variants, is_replaces_id_unique_violation, + readable_score_set_urns, refresh_variant_urns, ) from mavedb.lib.score_sets import ( @@ -77,20 +84,20 @@ from mavedb.lib.target_genes import find_or_create_target_gene_by_accession, find_or_create_target_gene_by_sequence from mavedb.lib.taxonomies import find_or_create_taxonomy from mavedb.lib.types.authentication import UserData +from mavedb.lib.urn_redirects import record_urn_redirect from mavedb.lib.urns import ( generate_experiment_set_urn, generate_experiment_urn, generate_score_set_urn, + lock_urn_assignment, ) -from mavedb.lib.workflow.pipeline_factory import PipelineFactory -from mavedb.models.clinical_control import ClinicalControl +from mavedb.lib.validation.exceptions import ValidationError as MaveDBValidationError +from mavedb.lib.variant_detail import load_variant_detail_inputs +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession @@ -101,19 +108,21 @@ ACCESS_CONTROL_ERROR_RESPONSES, BASE_400_RESPONSE, BASE_409_RESPONSE, + BASE_RESPONSES, GATEWAY_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) -from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set +from mavedb.view_models import clinical_control, gnomad_variant, score_set from mavedb.view_models.contributor import ContributorCreate from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.doi_identifier import DoiIdentifierCreate +from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.publication_identifier import PublicationIdentifierCreate from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse from mavedb.view_models.target_gene import TargetGeneCreate -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Score Sets" logger = logging.getLogger(__name__) @@ -122,37 +131,6 @@ SCORE_SET_SEARCH_MAX_PUBLICATION_IDENTIFIERS = 40 -async def enqueue_pipeline_entrypoint( - *, - db: Session, - worker: ArqRedis, - pipeline_name: str, - creating_user: Any, - pipeline_params: dict[str, Any], -) -> None: - pipeline_factory = PipelineFactory(session=db) - pipeline: Optional[Pipeline] = None - - try: - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=creating_user, - pipeline_params=pipeline_params, - ) - job = await worker.enqueue_job( - pipeline_entrypoint.job_function, pipeline_entrypoint.id, _job_id=arq_job_id(pipeline_entrypoint) - ) - except Exception: - pipeline_factory.discard_pipeline(pipeline) - raise - - if job is None: - logger.info(msg=f"Pipeline entrypoint for {pipeline_name} has already been enqueued.", extra=logging_context()) - else: - save_to_logging_context({"worker_job_id": job.job_id}) - logger.info(msg=f"Enqueued pipeline entrypoint for {pipeline_name}.", extra=logging_context()) - - async def enqueue_variant_creation( *, item: ScoreSet, @@ -220,15 +198,13 @@ async def enqueue_variant_creation( ) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="validate_map_annotate_score_set", - creating_user=user_data.user, - pipeline_params={ - "correlation_id": correlation_id_for_context(), - "score_set_id": item.id, - "updater_id": user_data.user.id, + score_set=item, + user=user_data.user, + extra_params={ "scores_file_key": scores_file_key, "counts_file_key": counts_file_key, "score_columns_metadata": item.dataset_columns.get("score_columns_metadata") @@ -622,13 +598,15 @@ def _score_set_response(item: ScoreSet, principal: Principal) -> score_set.Score """ Serialize a score set for a response, withholding the sub-resources this caller may not read. - Every route in this module that returns a ``ScoreSet`` view model builds it here. The two sub-resources - a score set carries have READ rules stricter than its own, and each was leaked from a different route - before this was centralized: + Every route in this module that returns a ``ScoreSet`` view model builds it here. The sub-resources a + score set carries have READ rules of their own, and each was leaked from a route before this was + centralized: - Calibrations. Publishing a score set does not publish its calibrations, and owning a score set does not entitle its owner to a community calibration someone else attached to it. - The superseding score set, which is usually still private while the score set it replaces is public. + - The score sets that meta-analyze this one. Anyone may start a meta-analysis of a published score set. + - Official collections, and the other members they list. The search routes are the deliberate exception: they answer with ``ShortScoreSet``, which carries neither sub-resource, so there is nothing for this function to narrow. Any route that widens its response model @@ -657,10 +635,19 @@ def _score_set_response(item: ScoreSet, principal: Principal) -> score_set.Score has_permission(principal.user_data, item.superseding_score_set, Action.READ).permitted ) + readable_meta_analysis_urns = readable_score_set_urns( + principal.user_data, + item.meta_analyzed_by_score_sets, # type: ignore[attr-defined] # backref of meta_analyzes_score_sets + ) + validated_item = score_set.ScoreSet.model_validate(item) return validated_item.model_copy( update={ "experiment": enrich_experiment_with_num_score_sets(item.experiment, principal.user_data), + "meta_analyzed_by_score_set_urns": [ + urn for urn in validated_item.meta_analyzed_by_score_set_urns if urn in readable_meta_analysis_urns + ], + "official_collections": readable_official_collections(principal.user_data, item.official_collections), "score_calibrations": [ calibration for calibration in (validated_item.score_calibrations or []) @@ -929,9 +916,17 @@ async def show_score_set( def get_score_set_csv_namespaces( *, urn: str, + response: Response, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the offered namespaces as they stood at this instant, so discovery matches an " + "`as_of` download. ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), ) -> Any: """ List the CSV column namespaces this score set has data for, labeled and grouped for a picker. @@ -964,10 +959,195 @@ def get_score_set_csv_namespaces( assert_permission(user_data, score_set, Action.READ) + # Echoed like the CSV endpoints: the instant discovery answered for is part of the answer. + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" return available_score_set_csv_namespaces( db, score_set, viewer=principal.viewer_for(ScoreCalibrationViewer), + as_of=as_of, + ) + + +@router.get( + "/score-sets/{urn}/variants", + status_code=200, + response_model=List[LeanVariant], + response_model_exclude_none=True, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Get the lean whole-set variant view for a score set", +) +async def get_score_set_lean_variants( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (mapping, allele links, VEP consequence) as it stood at " + "this instant, over the score set's fixed scores. ISO 8601, ideally timezone-aware. This is " + "content valid-time only — it never re-selects a score-set version. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + Return the lean whole-set view for a score set: one pre-chewed record per variant carrying the + selection key (variant URN), score, a representative consequence, the bridge identifiers into the + annotation dimensions (ClinGen allele id, assay-level digest), and the DNA + protein parsed + position/ref/alt blocks that drive the heatmap's level toggle. + + The full set is returned in one payload — the score-set page bins/sorts/filters across every + variant client-side. as_of time-travels the annotation layer only (scores are immutable); the + resolved value is echoed in the X-As-Of response header so the content-time is a visible fact. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "lean-variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + return get_lean_score_set_variants(db, score_set, as_of=as_of) + + +def _stream_score_set_variant_details( + db: Session, + variants: Sequence[Variant], + superseding_score_set: Optional[ScoreSet], + visible_calibration_ids: set[int], + *, + as_of: Optional[datetime] = None, +): + """Serialize the whole-set variant-detail export as NDJSON — one :class:`VariantDetail` per line. + + Each variant's detail envelope (the flat assay fields + ``preMapped``/``postMapped`` VRS pair + the + spec-pure GA4GH CategoricalVariant + the digest-keyed VEP/gnomAD/ClinVar annotation map) is assembled + from inputs loaded a chunk of variants at a time, so a large score set streams in a fixed number of + queries per chunk rather than building every envelope up front. ``superseding_score_set`` / + ``visible_calibration_ids`` are resolved once for the whole set. + """ + for variant, inputs in with_chunk_inputs( + variants, + lambda chunk: load_variant_detail_inputs( + db, chunk, visible_calibration_ids=visible_calibration_ids, as_of=as_of + ), + ): + detail = inputs.detail_for(variant, superseding_score_set=superseding_score_set) + # detail_for returns the lib transit dataclass; coerce it through the view model. + # exclude_none=False keeps a stable key set per line for programmatic consumers. + yield VariantDetail.model_validate(detail).model_dump_json(by_alias=True, exclude_none=False) + "\n" + + +@router.get( + "/score-sets/{urn}/variant-details", + status_code=200, + responses={ + 200: { + "content": {"application/x-ndjson": {}}, + "description": ( + "Newline-delimited JSON: one VariantDetail per mapped variant — the same envelope the " + "single-variant GET /variants/{urn} route serves, carrying the flat preMapped/postMapped " + "VRS pair, the spec-pure GA4GH CategoricalVariant, and the digest-keyed VEP/gnomAD/ClinVar " + "annotation map." + ), + }, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Download a score set's variant details (VRS + Cat-VRS + annotations)", +) +async def get_score_set_variant_details( + *, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (VRS, Cat-VRS membership + VEP/gnomAD/ClinVar annotations) " + "as it stood at this instant, over the score set's fixed scores. ISO 8601, ideally " + "timezone-aware. Content valid-time only — it never re-selects a score-set version. Defaults " + "to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """Download the score set's variant details — the whole-set streaming pair of the single-variant + ``GET /variants/{urn}`` detail endpoint, and the substrate-faithful replacement for the retired + ``/mapped-variants`` export. + + One record per *mapped* variant (unmapped variants carry no VRS and are omitted): the same + VariantDetail envelope the single-variant route serves — the flat ``preMapped``/``postMapped`` VRS + pair for VRS consumers, plus the spec-pure GA4GH CategoricalVariant and the digest-keyed + VEP/gnomAD/ClinVar annotation map for the full molecular picture. + + Streamed as NDJSON (like the annotated-variant exports) so a large score set downloads without + materializing every envelope server-side and a client can process it line by line. ``as_of`` + time-travels the molecular layer only (scores/classifications are immutable); the resolved value is + echoed in ``X-As-Of`` and the variant count in ``X-Total-Count``. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "variant-details", "as_of": as_of}) + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + + # Resolved here rather than inherited: fetch_score_set_by_urn returns the score set as it actually + # is, leaving narrowing to each response path. Mirrors the single-variant route, which must agree + # with this one -- they serve the same envelope. + superseding = score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + # A calibration's READ rule is stricter than its score set's, so reading the score set does not + # entitle a caller to a private calibration's classifications. + visible_calibration_ids = { + sc.id + for sc in score_set.score_calibrations + if sc.id is not None and has_permission(user_data, sc, Action.READ).permitted + } + variants = get_annotatable_variants(db, score_set, as_of=as_of) + return StreamingResponse( + _stream_score_set_variant_details( + db, + variants, + superseding, + visible_calibration_ids, + as_of=as_of, + ), + media_type="application/x-ndjson", + headers={ + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), + "X-Processing-Started": datetime.now().isoformat(), + "X-Stream-Type": "variant-detail", + "Access-Control-Expose-Headers": "X-As-Of, X-Total-Count, X-Processing-Started, X-Stream-Type", + }, + ) + + +@router.get( + "/score-sets/{urn}/mapped-variants", + status_code=410, + deprecated=True, + responses={410: BASE_RESPONSES[410]}, + summary="Removed; see GET /score-sets/{urn}/variant-details", +) +def get_score_set_mapped_variants_removed(*, urn: str) -> Any: + """This endpoint has been permanently removed. + + Its JSON-array response has been replaced by a streaming NDJSON payload with a different + field shape (flat ``preMapped``/``postMapped`` VRS pair rather than a ``MappedVariant``-keyed + record), so the two are not wire-compatible and this route does not redirect. Use + ``GET /score-sets/{urn}/variant-details`` instead. + """ + successor = f"/score-sets/{urn}/variant-details" + record_deprecated_usage(f"/score-sets/{urn}/mapped-variants", successor=successor) + # No wire-compatible successor (the response shape and cardinality changed), so no rel="successor-version" + # Link — the Warning points at the nearest replacement instead. + raise HTTPException( + status_code=410, + detail=(f"GET /score-sets/{urn}/mapped-variants has been removed. Use GET {successor} instead."), + headers=deprecation_headers( + successor=None, + warning=f"Removed; use GET {successor} (NDJSON, different field shape).", + sunset=MAPPED_VARIANT_SUNSET, + ), ) @@ -1002,6 +1182,14 @@ def get_score_set_variants_csv( include_custom_columns: Optional[bool] = Query( default=None, deprecated=True, description=INCLUDE_CUSTOM_COLUMNS_DESCRIPTION ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) as it stood at this " + "instant, over the variant's immutable submitted HGVS/scores/counts. ISO 8601, ideally " + "timezone-aware. No effect on the scores/counts namespaces. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), @@ -1061,6 +1249,8 @@ def get_score_set_variants_csv( "resource_property": "scores", "start": start, "limit": limit, + "drop_na_columns": drop_na_columns, + "as_of": as_of, "drop_unused_hgvs_columns": drop_unused_hgvs_columns, } ) @@ -1079,19 +1269,34 @@ def get_score_set_variants_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv( - db, - score_set, - namespaces, - True, - start, - limit, - drop_unused_hgvs_columns, - # Asked separately from the score set: a private calibration is readable only by its owner, - # investigator contributors, or an admin, whoever can read the score set. - viewer=principal.viewer_for(ScoreCalibrationViewer), + # Full-table CSV builds legitimately outrun the API's default statement timeout. + allow_long_statements(db) + + with csv_build_slot(rows_to_build(score_set.num_variants, start, limit)): + csv_str = get_score_set_variants_as_csv( + db, + score_set, + namespaces, + namespaced=True, + start=start, + limit=limit, + drop_unused_hgvs_columns_flag=drop_unused_hgvs_columns, + as_of=as_of, + # Asked separately from the score set: a private calibration is readable only by its owner, + # investigator contributors, or an admin, whoever can read the score set. + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) + # Both: the deprecation notice (when a legacy parameter was used) and the resolved content-time, + # so a CSV's as-of instant is a visible fact rather than something the caller has to remember. + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={ + **deprecated.response_headers, + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "Access-Control-Expose-Headers": "X-As-Of", + }, ) - return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1154,11 +1359,15 @@ def get_score_set_scores_csv( assert_permission(user_data, score_set, Action.READ) + # Full-table CSV builds legitimately outrun the API's default statement timeout. + allow_long_statements(db) + # Both score namespaces: this endpoint has always returned every score column the investigator # uploaded, and `scores` alone is now just the required one. - csv_str = get_score_set_variants_as_csv( - db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns - ) + with csv_build_slot(rows_to_build(score_set.num_variants, start, limit)): + csv_str = get_score_set_variants_as_csv( + db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns + ) return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @@ -1176,7 +1385,7 @@ def get_score_set_scores_csv( }, summary="Get score set counts in CSV format", ) -async def get_score_set_counts_csv( +def get_score_set_counts_csv( *, urn: str, start: int = Query(default=None, description="Start index for pagination"), @@ -1222,60 +1431,21 @@ async def get_score_set_counts_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) + # Full-table CSV builds legitimately outrun the API's default statement timeout. + allow_long_statements(db) - -@router.get( - "/score-sets/{urn}/mapped-variants", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Get mapped variants from score set by URN", -) -def get_score_set_mapped_variants( - *, - urn: str, - db: Session = Depends(deps.get_db), - user_data: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Return mapped variants from a score set, identified by URN. - """ - save_to_logging_context({"requested_resource": urn, "resource_property": "mapped-variants"}) - - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() - if not score_set: - logger.info( - msg="Could not fetch the requested mapped variants; No such score set exist.", extra=logging_context() - ) - raise HTTPException(status_code=404, detail=f"score set with URN {urn} not found") - - assert_permission(user_data, score_set, Action.READ) - - mapped_variants = ( - db.query(MappedVariant) - .filter(ScoreSet.urn == urn) - .filter(ScoreSet.id == Variant.score_set_id) - .filter(Variant.id == MappedVariant.variant_id) - .all() - ) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variant associated with score set URN {urn} was found", + with csv_build_slot(rows_to_build(score_set.num_variants, start, limit)): + csv_str = get_score_set_variants_as_csv( + db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns ) - - return mapped_variants + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) def _annotation_stream_record( - mapped_variant, annotation_function + inputs: AnnotationContextInputs, variant, annotation_function ) -> tuple[dict, Literal["annotated", "unannotated", "errored"]]: """ - Build the NDJSON record for one mapped variant, and classify its outcome. + Build the NDJSON record for one variant, and classify its outcome. Returns the record together with one of ``annotated``, ``unannotated``, or ``errored``. Nothing raises: a failure past the first record would end the body mid-stream, and since the 200 and its headers went @@ -1284,14 +1454,17 @@ def _annotation_stream_record( failing serialization. A variant with nothing to annotate is *not* an error. It is an expected absence, reported as a null - annotation. Which exceptions mean that is defined once, in ``EXPECTED_ABSENCE_EXCEPTIONS``, because - the corpus sweep has to draw the same line and the two must not drift apart. + annotation: either no annotation context could be built from the mapping substrate at ``as_of``, or + the builder declined the variant. Which exceptions mean that is defined once, in + ``EXPECTED_ABSENCE_EXCEPTIONS``, because the corpus sweep has to draw the same line and the two must + not drift apart. """ - variant_urn = mapped_variant.variant.urn + variant_urn = variant.urn try: - annotation = annotation_function(mapped_variant) - annotation_data = annotation.model_dump(exclude_none=True) if annotation else None + context = inputs.context_for(variant) + annotation = annotation_function(context) if context is not None else None + annotation_data = serialize_annotation(annotation) if annotation else None except EXPECTED_ABSENCE_EXCEPTIONS: logger.debug(f"Nothing to annotate for variant {variant_urn}.") return {"variant_urn": variant_urn, "annotation": None}, "unannotated" @@ -1312,16 +1485,19 @@ def _annotation_stream_record( return {"variant_urn": variant_urn, "annotation": annotation_data}, "annotated" -def _stream_generated_annotations(mapped_variants, annotation_function): +def _stream_generated_annotations(db, variants, annotation_function, as_of=None): """ Generator function to stream annotations as pure NDJSON data. + Builds each variant's :class:`VariantAnnotationContext` from the mapping-record substrate and passes it + to ``annotation_function``; variants with no live mapping data yield a null annotation. + Metadata should be provided via HTTP headers: - X-Total-Count: Total number of variants - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Type of annotation being streamed - Emits exactly one record per mapped variant, so a body holding fewer lines than ``X-Total-Count`` is + Emits exactly one record per annotatable variant, so a body holding fewer lines than ``X-Total-Count`` is a truncated one. Outcome counts are logged rather than appended to the body, which keeps every line a variant record and keeps this format identical to the public dump's ``va/{urn}.va.ndjson``. @@ -1329,13 +1505,15 @@ def _stream_generated_annotations(mapped_variants, annotation_function): consumed via Server-Sent Events if needed. """ start_time = time.time() - total_variants = len(mapped_variants) + total_variants = len(variants) processed_count = 0 outcome_counts = {"annotated": 0, "unannotated": 0, "errored": 0} - logger.info(f"Starting streaming processing of {total_variants} mapped variants") + logger.info(f"Starting streaming processing of {total_variants} variants") - for mv in mapped_variants: - result, outcome = _annotation_stream_record(mv, annotation_function) + for variant, inputs in with_chunk_inputs( + variants, lambda chunk: load_annotation_context_inputs(db, chunk, as_of=as_of) + ): + result, outcome = _annotation_stream_record(inputs, variant, annotation_function) outcome_counts[outcome] += 1 yield json.dumps(result, default=str) + "\n" @@ -1384,11 +1562,11 @@ def _stream_generated_annotations(mapped_variants, annotation_function): status_code=200, response_model=dict[str, Optional[VariantPathogenicityStatement]], response_model_exclude_none=True, - summary="Get pathogenicity statement annotations for mapped variants within a score set", + summary="Get pathogenicity statement annotations for variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream pathogenicity statement annotations for mapped variants.", + "description": "Stream pathogenicity statement annotations for variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1396,6 +1574,15 @@ def _stream_generated_annotations(mapped_variants, annotation_function): def get_score_set_annotated_variants( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), @@ -1403,16 +1590,16 @@ def get_score_set_annotated_variants( """ Retrieve annotated variants with pathogenicity statements for a given score set. - This endpoint streams pathogenicity evidence lines for all current mapped variants + This endpoint streams pathogenicity evidence lines for all current annotated variants associated with a specific score set. The response is returned as newline-delimited JSON (NDJSON) format for efficient processing of large datasets. NDJSON Response Format: - Each line corresponds to a mapped variant and contains a JSON object with the following - structure: + Each line in the response corresponds to an annotated variant and contains a JSON + object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Pathogenicity evidence line details } @@ -1424,7 +1611,7 @@ def get_score_set_annotated_variants( truncating the stream, and carries an additional `error` object: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": null, "error": {"type": "", "detail": ""} } @@ -1442,23 +1629,27 @@ def get_score_set_annotated_variants( Returns: Any: StreamingResponse containing newline-delimited JSON with pathogenicity - evidence lines for each mapped variant. Response includes headers with + evidence lines for each annotated variant. Response includes headers with total count, processing start time, and stream type information. + A score set that exists but has no annotatable variants (never mapped, or none live at ``as_of``) + streams an empty body with ``X-Total-Count: 0`` — an empty collection, not a 404. + Raises: HTTPException: 404 error if the score set with the given URN is not found. - HTTPException: 404 error if no mapped variants are associated with the score set. HTTPException: 403 error if the user lacks READ permissions for the score set. Note: This function logs the request context and validates user permissions before - processing. Only current (non-historical) mapped variants are included in - the response. + processing. Use the `as_of` parameter to reconstruct the molecular layer as it stood at a specific + instant, over the variant's fixed score. The response is streamed to allow for efficient handling + of large datasets, and progress updates are logged for monitoring purposes. """ save_to_logging_context( { "requested_resource": urn, "resource_property": "annotated-variants/pathogenicity-statement", + "as_of": as_of, } ) @@ -1472,20 +1663,19 @@ def get_score_set_annotated_variants( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct evidence lines.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, partial(variant_pathogenicity_statement, principal=principal)), + _stream_generated_annotations( + db, variants, partial(variant_pathogenicity_statement, principal=principal), as_of=as_of + ), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "pathogenicity-evidence-line", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1498,11 +1688,11 @@ def get_score_set_annotated_variants( status_code=200, response_model=dict[str, Optional[Statement]], response_model_exclude_none=True, - summary="Get functional impact statement annotations for mapped variants within a score set", + summary="Get functional impact statement annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional impact statement annotations for mapped variants.", + "description": "Stream functional impact statement annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1510,6 +1700,15 @@ def get_score_set_annotated_variants( def get_score_set_annotated_variants_functional_statement( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), @@ -1517,16 +1716,16 @@ def get_score_set_annotated_variants_functional_statement( """ Retrieve functional impact statements for annotated variants in a score set. - This endpoint streams functional impact statements for all current mapped variants + This endpoint streams functional impact statements for all current annotated variants associated with a specific score set. The response is delivered as newline-delimited JSON (NDJSON) format. NDJSON Response Format: - Each line corresponds to a mapped variant and contains a JSON object with the following - structure: + Each line in the response corresponds to an annotated variant and contains a JSON + object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional impact statement details } @@ -1538,7 +1737,7 @@ def get_score_set_annotated_variants_functional_statement( truncating the stream, and carries an additional `error` object: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": null, "error": {"type": "", "detail": ""} } @@ -1554,20 +1753,28 @@ def get_score_set_annotated_variants_functional_statement( Returns: StreamingResponse: NDJSON stream containing functional impact statements for each - mapped variant. Response includes headers with total count, processing start time, + annotated variant. Response includes headers with total count, processing start time, and stream type information. Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Note: - Only current (non-historical) mapped variants are included in the response. - The function requires appropriate read permissions on the score set. + The function requires appropriate read permissions on the score set. Use the `as_of` + parameter to reconstruct the molecular layer as it stood at a specific instant, over + the variant's fixed score. The response is streamed to allow for efficient handling of + large datasets, and progress updates are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/functional-statement"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/functional-statement", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1579,22 +1786,19 @@ def get_score_set_annotated_variants_functional_statement( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct functional impact statements.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( _stream_generated_annotations( - mapped_variants, partial(variant_functional_impact_statement, principal=principal) + db, variants, partial(variant_functional_impact_statement, principal=principal), as_of=as_of ), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-impact-statement", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1607,11 +1811,11 @@ def get_score_set_annotated_variants_functional_statement( status_code=200, response_model=dict[str, Optional[ExperimentalVariantFunctionalImpactStudyResult]], response_model_exclude_none=True, - summary="Get functional study result annotations for mapped variants within a score set", + summary="Get functional study result annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional study result annotations for mapped variants.", + "description": "Stream functional study result annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1619,22 +1823,31 @@ def get_score_set_annotated_variants_functional_statement( def get_score_set_annotated_variants_functional_study_result( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ): """ Retrieve functional study results for annotated variants in a score set. - This endpoint streams functional study result annotations for all current mapped variants + This endpoint streams functional study result annotations for all current annotated variants associated with a specific score set. The results are returned as newline-delimited JSON (NDJSON) format for efficient streaming of large datasets. NDJSON Response Format: - Each line corresponds to a mapped variant and contains a JSON object with the following - structure: + Each line in the response corresponds to a annotated variant and contains a JSON + object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional study result details } @@ -1646,7 +1859,7 @@ def get_score_set_annotated_variants_functional_study_result( truncating the stream, and carries an additional `error` object: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": null, "error": {"type": "", "detail": ""} } @@ -1663,7 +1876,7 @@ def get_score_set_annotated_variants_functional_study_result( Returns: StreamingResponse: A streaming response containing functional study results in NDJSON format. Headers include: - - X-Total-Count: Total number of mapped variants being streamed + - X-Total-Count: Total number of annotated variants being streamed - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Set to "functional-study-result" - Access-Control-Expose-Headers: Exposed headers for CORS @@ -1671,15 +1884,22 @@ def get_score_set_annotated_variants_functional_study_result( Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Notes: - - Only returns current mapped variants (MappedVariant.current == True) - - Eagerly loads related ScoreSet data including publications, users, license, and experiment - - Logs requests and errors for monitoring and debugging purposes + - The `as_of` parameter allows reconstruction of the molecular layer as it stood at a specific + instant, over the variant's fixed score. It is ISO 8601 formatted and ideally timezone-aware. + - The response is streamed to allow for efficient handling of large datasets, and progress updates + are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/study-result"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/study-result", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1691,20 +1911,17 @@ def get_score_set_annotated_variants_functional_study_result( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct study results.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_study_result), + _stream_generated_annotations(db, variants, variant_study_result, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-study-result", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1926,7 +2143,7 @@ async def create_score_set( score_calibrations: list[ScoreCalibration] = [] if item_create.score_calibrations: - for calibration_create in item_create.score_calibrations: + for calibration_index, calibration_create in enumerate(item_create.score_calibrations): # TODO#592: Support for class-based calibrations on score set creation if calibration_create.class_based: logger.info( @@ -1938,9 +2155,34 @@ async def create_score_set( detail="Class-based calibrations are not supported on score set creation. Please create class-based calibrations after creating the score set.", ) - created_calibration_item = await create_score_calibration( - db, calibration_create, user_data.user, variant_classes=None - ) + # TODO#592: Support for controls on score set creation + if calibration_create.controls: + logger.info( + msg="Failed to create score set; Calibration controls are not supported on score set creation.", + extra=logging_context(), + ) + raise HTTPException( + status_code=409, + detail="Calibration controls are not supported on score set creation. Please add controls after creating the score set.", + ) + + try: + created_calibration_item = await create_score_calibration( + db, calibration_create, user_data.user, variant_classes=None + ) + except MaveDBValidationError as e: + # custom_loc is relative to the calibration body; nest it under this calibration. + field_loc = (e.custom_loc or ["body"])[1:] + raise HTTPException( + status_code=422, + detail=[ + { + "loc": ["body", "scoreCalibrations", calibration_index, *field_loc], + "msg": str(e), + "type": "value_error", + } + ], + ) created_calibration_item.investigator_provided = True # necessarily true on score set creation score_calibrations.append(created_calibration_item) @@ -2312,30 +2554,22 @@ async def update_score_set_with_variants( # get existing item from db existing_item = db.query(ScoreSet).filter(ScoreSet.urn == urn).one_or_none() - # merge existing item data with item_update data to validate against ScoreSetUpdate - - if existing_item: - existing_item_data = score_set.ScoreSet.model_validate(existing_item).model_dump() - updated_data = {**existing_item_data, **item_update_partial.model_dump(exclude_unset=True)} - try: - score_set.ScoreSetUpdate.model_validate(updated_data) - except ValidationError as e: - # format as fastapi request validation error - raise RequestValidationError(errors=e.errors()) - else: + if not existing_item: logger.info(msg="Failed to update score set; The requested score set does not exist.", extra=logging_context()) raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found") - itemUpdateResult = await score_set_update( - db=db, - urn=urn, - item_update=item_update_partial, - exclude_unset=True, - user_data=user_data, - existing_item=existing_item, - ) - updatedItem = itemUpdateResult["item"] - should_create_variants = itemUpdateResult.get("should_create_variants", False) + # Before validation, whose errors echo the existing record's values: a caller who may not update this + # score set must not learn anything about it, including that it exists. + assert_permission(user_data, existing_item, Action.UPDATE) + + # merge existing item data with item_update data to validate against ScoreSetUpdate + existing_item_data = score_set.ScoreSet.model_validate(existing_item).model_dump() + updated_data = {**existing_item_data, **item_update_partial.model_dump(exclude_unset=True)} + try: + score_set.ScoreSetUpdate.model_validate(updated_data) + except ValidationError as e: + # format as fastapi request validation error + raise RequestValidationError(errors=e.errors()) existing_score_columns_metadata = (existing_item.dataset_columns or {}).get("score_columns_metadata", {}) existing_count_columns_metadata = (existing_item.dataset_columns or {}).get("count_columns_metadata", {}) @@ -2346,15 +2580,32 @@ async def update_score_set_with_variants( did_count_columns_metadata_change = ( dataset_column_metadata.get("count_columns_metadata", {}) != existing_count_columns_metadata ) + request_sets_scores = ( + did_score_columns_metadata_change + or did_count_columns_metadata_change + or any([val is not None for val in score_set_variants_data.values()]) + ) + + # Checked before the update is applied, not just before the job is enqueued: score_set_update + # commits the metadata half of this request, so refusing afterwards would leave a caller who may + # not set scores with a half-applied update and no way to tell which half landed. + if request_sets_scores: + assert_permission(user_data, existing_item, Action.SET_SCORES) + + itemUpdateResult = await score_set_update( + db=db, + urn=urn, + item_update=item_update_partial, + exclude_unset=True, + user_data=user_data, + existing_item=existing_item, + ) + updatedItem = itemUpdateResult["item"] + should_create_variants = itemUpdateResult.get("should_create_variants", False) # run variant creation job only if targets have changed (indicated by "should_create_variants"), new score # or count files were uploaded, or dataset column metadata has changed - if ( - should_create_variants - or did_score_columns_metadata_change - or did_count_columns_metadata_change - or any([val is not None for val in score_set_variants_data.values()]) - ): + if should_create_variants or request_sets_scores: assert_permission(user_data, updatedItem, Action.SET_SCORES) updatedItem.processing_state = ProcessingState.processing @@ -2439,6 +2690,11 @@ async def update_score_set( should_create_variants = itemUpdateResult["should_create_variants"] if should_create_variants: + # Structurally unreachable for a published score set, since score_set_update only sets this + # flag while the score set is private. Asserted anyway so every path that recreates variants + # states the same requirement, and a future change to that flag cannot open a hole here. + assert_permission(user_data, updatedItem, Action.SET_SCORES) + # Although this is also updated within the variant creation job, update it here # as well so that we can display the proper UI components (queue invocation delay # races the score set GET request). @@ -2542,6 +2798,9 @@ async def publish_score_set( """ save_to_logging_context({"requested_resource": urn}) + # Taken before loading the score set so a concurrent publish of the same record waits, then sees it published. + lock_urn_assignment(db) + item: Optional[ScoreSet] = db.query(ScoreSet).filter(ScoreSet.urn == urn).one_or_none() if not item: logger.info(msg="Failed to publish score set; The requested score set does not exist.", extra=logging_context()) @@ -2549,6 +2808,13 @@ async def publish_score_set( assert_permission(user_data, item, Action.PUBLISH) + if not item.private: + logger.info( + msg="Failed to publish score set; The requested score set has already been published.", + extra=logging_context(), + ) + raise HTTPException(status_code=409, detail="This score set has already been published.") + if not item.experiment: logger.info( msg="Failed to publish score set; The requested score set does not belong to an experiment.", @@ -2579,46 +2845,67 @@ async def publish_score_set( detail="cannot publish score set without variant scores", ) - published_date = date.today() + # Autoflush can write the new URNs at any query below, so a collision may surface before the commit. + try: + published_date = date.today() - if item.experiment.experiment_set.private or not item.experiment.experiment_set.published_date: - item.experiment.experiment_set.urn = generate_experiment_set_urn(db) - item.experiment.experiment_set.private = False - item.experiment.experiment_set.published_date = published_date - db.add(item.experiment.experiment_set) + if item.experiment.experiment_set.private or not item.experiment.experiment_set.published_date: + retired_experiment_set_urn = item.experiment.experiment_set.urn + item.experiment.experiment_set.urn = generate_experiment_set_urn(db) + record_urn_redirect(db, retired_experiment_set_urn, item.experiment.experiment_set.urn) + item.experiment.experiment_set.private = False + item.experiment.experiment_set.published_date = published_date + db.add(item.experiment.experiment_set) - save_to_logging_context({"experiment_set": item.experiment.experiment_set.urn}) + save_to_logging_context({"experiment_set": item.experiment.experiment_set.urn}) - if item.experiment.private or not item.experiment.published_date: - item.experiment.urn = generate_experiment_urn( - db, - item.experiment.experiment_set, - experiment_is_meta_analysis=len(item.meta_analyzes_score_sets) > 0, - ) - item.experiment.private = False - item.experiment.published_date = published_date - db.add(item.experiment) + if item.experiment.private or not item.experiment.published_date: + retired_experiment_urn = item.experiment.urn + item.experiment.urn = generate_experiment_urn( + db, + item.experiment.experiment_set, + experiment_is_meta_analysis=len(item.meta_analyzes_score_sets) > 0, + ) + record_urn_redirect(db, retired_experiment_urn, item.experiment.urn) + item.experiment.private = False + item.experiment.published_date = published_date + db.add(item.experiment) - save_to_logging_context({"experiment": item.experiment.urn}) + save_to_logging_context({"experiment": item.experiment.urn}) - item.urn = generate_score_set_urn(db, item.experiment) - item.private = False - item.published_date = published_date - refresh_variant_urns(db, item) + retired_score_set_urn = item.urn + item.urn = generate_score_set_urn(db, item.experiment) + # Variant URNs are rewritten below from the score set's, so this one redirect forwards them too. + record_urn_redirect(db, retired_score_set_urn, item.urn) + item.private = False + item.published_date = published_date + refresh_variant_urns(db, item) - save_to_logging_context({"score_set": item.urn}) + save_to_logging_context({"score_set": item.urn}) - db.add(item) - db.commit() + db.add(item) + db.commit() + except IntegrityError: + db.rollback() + logger.warning( + msg="Failed to publish score set; A newly assigned URN collided with an existing record.", + extra=logging_context(), + exc_info=True, + ) + raise HTTPException( + status_code=409, + detail="This score set could not be published because its new URN is already in use. " + "Please reload the page to check whether it was published, and try again if it was not.", + ) db.refresh(item) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="publish_score_set", - creating_user=user_data.user, - pipeline_params={"correlation_id": correlation_id_for_context(), "score_set_id": item.id}, + score_set=item, + user=user_data.user, ) except Exception as exc: logger.warning( @@ -2633,7 +2920,7 @@ async def publish_score_set( @router.get( "/score-sets/{urn}/clinical-controls", status_code=200, - response_model=list[clinical_control.ClinicalControlWithMappedVariants], + response_model=list[clinical_control.ClinicalControlWithClinvarLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get clinical controls for a score set", @@ -2641,20 +2928,33 @@ async def publish_score_set( async def get_clinical_controls_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. _db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> Sequence[ClinicalControl]: +) -> list[clinical_control.ClinicalControlWithClinvarLinks]: """ Fetch relevant clinical controls for a given score set. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_name = db db_version = version + if db_name is not None: + save_to_logging_context({"db_name": db_name}) + if db_version is not None: + save_to_logging_context({"db_version": db_version}) item: Optional[ScoreSet] = _db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2666,26 +2966,13 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinicalControl) - .join(ClinicalControl.mapped_variants) - .join(MappedVariant.variant) - .options(contains_eager(ClinicalControl.mapped_variants).contains_eager(MappedVariant.variant)) - .filter(MappedVariant.current.is_(True)) - .filter(Variant.score_set_id == item.id) + controls = get_clinical_controls_with_variant_urns( + _db, item.id, as_of=as_of, db_name=db_name, db_version=db_version ) - if db_name is not None: - save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_name == db_name) - - if db_version is not None: - save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_version == db_version) - - clinical_controls: Sequence[ClinicalControl] = _db.scalars(clinical_controls_query).unique().all() - - if not clinical_controls: + if not controls: + # Can legitimately fire even for a (db_name, db_version) sourced from `.../options`: liveness + # is re-evaluated per call, see that endpoint's docstring. logger.info( msg="No clinical control variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2695,9 +2982,27 @@ async def get_clinical_controls_for_score_set( detail=f"No clinical control variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(clinical_controls)}) + save_to_logging_context({"resource_count": len(controls)}) - return clinical_controls + return [ + clinical_control.ClinicalControlWithClinvarLinks.model_validate( + { + "id": ctrl.id, + "db_identifier": ctrl.db_identifier, + "gene_symbol": ctrl.gene_symbol, + "clinical_significance": ctrl.clinical_significance, + "clinical_review_status": ctrl.clinical_review_status, + "db_version": ctrl.db_version, + "db_name": ctrl.db_name, + "modification_date": ctrl.modification_date, + "creation_date": ctrl.creation_date, + "clinvar_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], + } + ) + for ctrl, links in controls + ] @router.get( @@ -2711,14 +3016,33 @@ async def get_clinical_controls_for_score_set( async def get_clinical_controls_options_for_score_set( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), ) -> list[dict[str, Union[str, list[str]]]]: """ Fetch clinical control options for a given score set. + + Each ``(db_name, db_version)`` pair returned here was live at the moment of this call, but + liveness is re-evaluated independently per request. A pair fetched here can have its backing + ``ClinvarAlleleLink`` retired before a later call to ``GET /score-sets/{urn}/clinical-controls`` + filters on it, in which case that call 404s. Pin an explicit ``as_of`` on both calls to avoid this + possibility. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_control_options"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "clinical_control_options", + "as_of": as_of, + } + ) item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2730,23 +3054,9 @@ async def get_clinical_controls_options_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinicalControl.db_name, ClinicalControl.db_version) - .join(MappedVariant, ClinicalControl.mapped_variants) - .join(Variant) - .where(MappedVariant.current.is_(True)) - .where(Variant.score_set_id == item.id) - ) - - clinical_controls_for_item = db.execute(clinical_controls_query).unique() - - # NOTE: We return options even for pairwise groupings which may have no associated mapped variants - # and 404 when ultimately requested together. - clinical_control_options: dict[str, list[str]] = {} - for db_name, db_version in clinical_controls_for_item: - clinical_control_options.setdefault(db_name, []).append(db_version) + options = get_clinical_control_options(db, item.id, as_of=as_of) - if not clinical_control_options: + if not options: logger.info( msg="Failed to fetch clinical control options for score set; No clinical control variants are associated with this score set.", extra=logging_context(), @@ -2756,16 +3066,13 @@ async def get_clinical_controls_options_for_score_set( detail=f"no clinical control variants associated with score set URN {urn} were found", ) - return [ - dict(zip(("db_name", "available_versions"), (db_name, db_versions))) - for db_name, db_versions in clinical_control_options.items() - ] + return [{"db_name": db_name, "available_versions": available_versions} for db_name, available_versions in options] @router.get( "/score-sets/{urn}/gnomad-variants", status_code=200, - response_model=list[gnomad_variant.GnomADVariantWithMappedVariants], + response_model=list[gnomad_variant.GnomADVariantWithVariantLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get gnomad variants for a score set", @@ -2773,16 +3080,26 @@ async def get_clinical_controls_options_for_score_set( async def get_gnomad_variants_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → gnomAD link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), version: Optional[str] = None, -) -> Sequence[GnomADVariant]: +) -> list[gnomad_variant.GnomADVariantWithVariantLinks]: """ - Fetch relevant gnomad variants for a given score set. + Fetch relevant gnomad variants for a given score set, each paired with the score-set variants (and + annotated allele digests) it links to over the allele substrate. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_version = version item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() @@ -2795,28 +3112,12 @@ async def get_gnomad_variants_for_score_set( assert_permission(user_data, item, Action.READ) - gnomad_variants_query = ( - select(GnomADVariant) - .join(MappedVariant, GnomADVariant.mapped_variants) - .join(Variant) - .where(Variant.score_set_id == item.id) - ) - if db_version is not None: save_to_logging_context({"db_version": db_version}) - gnomad_variants_query = gnomad_variants_query.where(GnomADVariant.db_version == db_version) - - gnomad_variants_for_item: Sequence[GnomADVariant] = db.scalars(gnomad_variants_query).all() - gnomad_variants_with_mapped_variant = [] - for gnomad_variant_in_item in gnomad_variants_for_item: - gnomad_variant_in_item.mapped_variants = [ - mv for mv in gnomad_variant_in_item.mapped_variants if mv.current and mv.variant.score_set_id == item.id - ] - if gnomad_variant_in_item.mapped_variants: - gnomad_variants_with_mapped_variant.append(gnomad_variant_in_item) + gnomad_variants = get_gnomad_variants_with_variant_urns(db, item.id, as_of=as_of, db_version=db_version) - if not gnomad_variants_with_mapped_variant: + if not gnomad_variants: logger.info( msg="No gnomad variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2826,6 +3127,26 @@ async def get_gnomad_variants_for_score_set( detail=f"No gnomad variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(gnomad_variants_for_item)}) + save_to_logging_context({"resource_count": len(gnomad_variants)}) - return gnomad_variants_for_item + return [ + gnomad_variant.GnomADVariantWithVariantLinks.model_validate( + { + "id": gv.id, + "db_name": gv.db_name, + "db_identifier": gv.db_identifier, + "db_version": gv.db_version, + "allele_count": gv.allele_count, + "allele_number": gv.allele_number, + "allele_frequency": gv.allele_frequency, + "faf95_max": gv.faf95_max, + "faf95_max_ancestry": gv.faf95_max_ancestry, + "creation_date": gv.creation_date, + "modification_date": gv.modification_date, + "variant_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], + } + ) + for gv, links in gnomad_variants + ] diff --git a/src/mavedb/routers/seqrepo.py b/src/mavedb/routers/seqrepo.py index 42ec14645..91d81c9a2 100644 --- a/src/mavedb/routers/seqrepo.py +++ b/src/mavedb/routers/seqrepo.py @@ -71,7 +71,31 @@ def get_sequence( status_code=400, detail=f"Multiple sequences exist for alias '{alias}'. Use an explicit namespace." ) - return StreamingResponse(sequence_generator(sr, seq_ids[0], start, end), media_type="text/plain") + seq_id = seq_ids[0] + seq_len = sr.sequences.fetch_seqinfo(seq_id)["len"] + + # Resolve to concrete half-open bounds so the check and the streamed body agree, even when only + # one of start/end is supplied. + seq_start = start if start is not None else 0 + seq_end = end if end is not None else seq_len + + # Mirrors refget.py's two-sided rule: start == seq_len (an empty tail slice) is legal. + if start is not None and end is not None: + if not 0 <= seq_start <= seq_end <= seq_len: + logger.error(msg="Invalid coordinates: range lies outside the sequence.", extra=logging_context()) + raise HTTPException( + status_code=422, detail=f"Invalid coordinates: must obey 0 <= start <= end <= {seq_len}" + ) + # Mirrors refget.py: a start beyond the sequence length is rejected. + elif start is not None: + if not 0 <= seq_start <= seq_len: + logger.error(msg="Invalid coordinates: start lies outside the sequence.", extra=logging_context()) + raise HTTPException(status_code=422, detail=f"Invalid coordinates: must obey 0 <= start <= {seq_len}") + # Mirrors refget.py: an over-long end is clamped rather than rejected. + elif end is not None: + seq_end = max(0, min(seq_end, seq_len)) + + return StreamingResponse(sequence_generator(sr, seq_id, seq_start, seq_end), media_type="text/plain") @router.get("/metadata/{alias}", response_model=SeqRepoMetadata, summary="Get sequence metadata by alias") diff --git a/src/mavedb/routers/shared.py b/src/mavedb/routers/shared.py index f98edb39c..2cddfd23d 100644 --- a/src/mavedb/routers/shared.py +++ b/src/mavedb/routers/shared.py @@ -8,6 +8,7 @@ 403: {"description": "Forbidden. Insufficient permissions."}, 404: {"description": "Resource not found."}, 409: {"description": "Conflict with current resource state."}, + 410: {"description": "Gone. This resource has been permanently removed."}, 416: {"description": "Requested range not satisfiable."}, 422: {"description": "Unprocessable entity. Validation failed."}, 429: {"description": "Too many requests. Rate limit exceeded."}, @@ -23,6 +24,7 @@ BASE_403_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {403: BASE_RESPONSES[403]} BASE_404_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {404: BASE_RESPONSES[404]} BASE_409_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {409: BASE_RESPONSES[409]} +BASE_410_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {410: BASE_RESPONSES[410]} BASE_416_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {416: BASE_RESPONSES[416]} BASE_422_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {422: BASE_RESPONSES[422]} BASE_429_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {429: BASE_RESPONSES[429]} diff --git a/src/mavedb/routers/statistics.py b/src/mavedb/routers/statistics.py index e52d2582d..5deea5636 100644 --- a/src/mavedb/routers/statistics.py +++ b/src/mavedb/routers/statistics.py @@ -345,7 +345,7 @@ def record_mapped_variant_counts(db: Session = Depends(get_db)) -> dict[str, int within a given record. """ variants = db.execute( - select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapped_variant_id)) + select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapping_record_id)) .group_by(PublishedVariantsMV.score_set_urn) .order_by(PublishedVariantsMV.score_set_urn) ).all() @@ -650,10 +650,10 @@ def mapped_variant_counts( """ # Fast path: total distinct mapped variants (optionally only current) without per-date aggregation. if group is None: - total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapped_variant_id))) + total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapping_record_id))) if onlyCurrent: - total_stmt = total_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + total_stmt = total_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) total = db.execute(total_stmt).scalar_one() # type: ignore return OrderedDict([("count", total)]) @@ -661,11 +661,11 @@ def mapped_variant_counts( # Grouped path: materialize distinct counts per published_date, then roll up. per_date_stmt = select( PublishedVariantsMV.published_date, - func.count(func.distinct(PublishedVariantsMV.mapped_variant_id)), + func.count(func.distinct(PublishedVariantsMV.mapping_record_id)), ) if onlyCurrent: - per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) per_date = db.execute( per_date_stmt.group_by(PublishedVariantsMV.published_date).order_by(PublishedVariantsMV.published_date) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index bb76716c4..79f4b92be 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,31 +1,40 @@ -import itertools import logging -import re -from typing import Any, List, Optional +from datetime import datetime +from typing import Annotated, Any, List, Optional -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Path, Query, Response from fastapi.exceptions import HTTPException -from fastapi.responses import StreamingResponse +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse, StreamingResponse +from ga4gh.core.identifiers import GA4GH_IR_REGEXP +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session, joinedload -from sqlalchemy.sql import or_ +from sqlalchemy.orm import Session from mavedb import deps +from mavedb.lib.alleles import find_variants_by_vrs_identifier +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.conformance import EmittedAnnotation, serialize_annotation +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authentication import get_current_user from mavedb.lib.authorization import get_principal from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr +from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData -from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet +from mavedb.lib.variant_detail import get_variant_detail from mavedb.models.variant import Variant -from mavedb.models.variant_translation import VariantTranslation from mavedb.routers.shared import ( ACCESS_CONTROL_ERROR_RESPONSES, BASE_400_RESPONSE, @@ -33,11 +42,8 @@ ROUTER_BASE_PREFIX, ) from mavedb.view_models.csv_namespace import AvailableCsvNamespace -from mavedb.view_models.variant import ( - ClingenAlleleIdVariantLookupResponse, - ClingenAlleleIdVariantLookupsRequest, - VariantEffectMeasurementWithScoreSet, -) +from mavedb.view_models.variant import VariantVrsMatch +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -56,418 +62,312 @@ } -@router.post( - "/variants/clingen-allele-id-lookups", +def _annotation_response(annotation: EmittedAnnotation, as_of: Optional[datetime]) -> JSONResponse: + """Emit a VA-Spec annotation with its nested evidence intact (see :func:`serialize_annotation`). + + Returning a response bypasses FastAPI's ``response_model`` serialization, which would drop the nested + evidence's subclass fields, so the ``X-As-Of`` header is set here rather than on the injected response. + """ + return JSONResponse( + content=jsonable_encoder(serialize_annotation(annotation, exclude_none=False)), + headers={"X-As-Of": as_of.isoformat() if as_of is not None else "current"}, + ) + + +def _fetch_readable_variant(db: Session, user_data: Optional[UserData], urn: str) -> Variant: + """Fetch a single variant by URN and assert the caller may read its score set. + + Raises 404 when no such variant exists, 500 when the URN is not unique (an invariant break), and the + standard 403 (via ``assert_permission``) when the score set is not readable. + """ + try: + variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + return variant + + +@router.get( + "/variants/vrs/{identifier}", status_code=200, - response_model=list[ClingenAlleleIdVariantLookupResponse], - responses={ - **BASE_400_RESPONSE, - **ACCESS_CONTROL_ERROR_RESPONSES, - }, - summary="Lookup variants by ClinGen Allele IDs", + response_model=list[VariantVrsMatch], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Look up variants by VRS identifier", ) -def lookup_variants( +def lookup_variants_by_vrs_identifier( *, - request: ClingenAlleleIdVariantLookupsRequest, + response: Response, + identifier: Annotated[ + str, + Path( + description="A GA4GH digest-based identifier for a variant's pre-mapped or post-mapped variation.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + regex=GA4GH_IR_REGEXP, + ), + ], + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), - user_data: UserData = Depends(get_current_user), -): + user_data: Optional[UserData] = Depends(get_current_user), +) -> list[VariantVrsMatch]: + """Resolve a GA4GH VRS identifier to the readable variants it identifies. + + The identifier may be a reference-level (post-mapped) allele or a variant's pre-mapped variation on + its submitted target sequence. A deduplicated allele may be shared across score sets, so one + identifier can resolve to several variants. This is a lookup returning a collection: results are filtered to the score sets the caller + may read, and an empty list is returned when nothing readable matches. An absent identifier and a + match visible only in a private score set are deliberately indistinguishable (both yield ``[]``), so + the response never reveals a private allele's existence. """ - Lookup variants by ClinGen Allele IDs. + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + matches = find_variants_by_vrs_identifier(db, identifier, as_of=as_of) + permitted = [ + (variant, allele) + for variant, allele in matches + if has_permission(user_data, variant.score_set, Action.READ).permitted + ] + + return [ + VariantVrsMatch( + variant_urn=variant.urn or "", + clingen_allele_id=allele.clingen_allele_id if allele is not None else None, + vrs_id=(allele.post_mapped or {}).get("id") if allele is not None else None, + level=allele.level if allele is not None else None, + ) + for variant, allele in permitted + ] + + +@router.get( + "/variants/{urn}", + status_code=200, + response_model=VariantDetail, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + # Emit nulls rather than omitting absent fields: this shape is a stable, self-describing envelope + # (its bulk pair, GET /score-sets/{urn}/variant-details, streams the same object one-per-line for + # dataframe/columnar consumption), so every response carries the same key set. + response_model_exclude_none=False, + summary="Fetch assayed variant detail by URN", +) +def get_variant( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """Fetch the two-tier detail envelope for a single assayed variant by URN. + + Flat assay-level fields for the common UI case plus the spec-pure GA4GH CategoricalVariant and a + digest-keyed annotation map for machine/standard consumers. A superseded variant is served (it is + the citable unit) but self-describes via isCurrent/supersededByScoreSet rather than reading as current. """ - save_to_logging_context({"requested_resource": "clingen-allele-id-lookups"}) - save_to_logging_context({"clingen_allele_ids_to_lookup": request.clingen_allele_ids}) - logger.debug(msg="Looking up variants by Clingen Allele IDs", extra=logging_context()) - - # sort multi-variant components lexicographically, as they are in the database - request.clingen_allele_ids = [",".join(sorted(allele_id.split(","))) for allele_id in request.clingen_allele_ids] - exact_match_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(request.clingen_allele_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - save_to_logging_context({"num_variants_matching_clingen_allele_ids": len(exact_match_variants)}) - logger.debug(msg="Found variants with exactly matching ClinGen Allele IDs", extra=logging_context()) - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - - variants_by_allele_id: dict[str, dict] = { - allele_id: { - "clingen_allele_id": allele_id, - "exact_match": {"clingen_allele_id": allele_id, "variant_effect_measurements": []}, - "equivalent_nt": [], - "equivalent_aa": [], - } - for allele_id in request.clingen_allele_ids + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + variant = _fetch_readable_variant(db, user_data, urn) + + # Version standing: resolve the superseding version's visibility exactly as fetch_score_set_by_urn + # does — a newer version the user cannot read is not leaked (the variant then reads as current). + superseding = variant.score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + visible_calibration_ids = { + sc.id + for sc in variant.score_set.score_calibrations + if sc.id is not None and has_permission(user_data, sc, Action.READ).permitted } - for variant, allele_id in exact_match_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - num_variants_matching_clingen_allele_ids_and_permitted += 1 - variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"].append(variant) - save_to_logging_context( - {"clingen_allele_ids_with_permitted_variants": num_variants_matching_clingen_allele_ids_and_permitted} + return get_variant_detail( + db, + variant, + superseding_score_set=superseding, + visible_calibration_ids=visible_calibration_ids, + as_of=as_of, ) - for allele_id in request.clingen_allele_ids: - if not variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"]: - variants_by_allele_id[allele_id]["exact_match"] = None - - for allele_id in request.clingen_allele_ids: - # validate and determine whether multi-variant - # NOTE: assuming we never have more than 2 components in a multi-variant - single_clingen_allele_id_re = r"^[CP]A\d+$" - multi_clingen_allele_id_re = r"^[CP]A\d+,[CP]A\d+$" - single_or_multi_clingen_allele_id_re = r"^[CP]A\d+(,[CP]A\d+)?$" - - if re.fullmatch(single_or_multi_clingen_allele_id_re, allele_id) is None: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}'", - ) - - if re.fullmatch(single_clingen_allele_id_re, allele_id): - if allele_id.startswith("PA"): - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where(VariantTranslation.aa_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == allele_id, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_aa_clingen_ids = [ - var_translation.aa_clingen_id - for var_translation in related_clingen_ids - if var_translation.aa_clingen_id != allele_id - ] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_clingen_ids = [var_translation.nt_clingen_id for var_translation in related_clingen_ids] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - elif allele_id.startswith("CA"): - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where(VariantTranslation.nt_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - related_aa_clingen_ids = [var_translation.aa_clingen_id for var_translation in related_clingen_ids] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_nt_clingen_ids = [ - var_translation.nt_clingen_id - for var_translation in related_clingen_ids - if var_translation.nt_clingen_id != allele_id - ] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - elif re.fullmatch(multi_clingen_allele_id_re, allele_id): - # validate each component allele id - already done via re - # allow more than 2 components? - # full matches are determined the same exact way as single variant (done above already) - # equivalent aa/nt: if all components of variant are represented by equivalent or exact match, but not all exact matches because those are already represented above - # so basically go through variant translations table and create every possible combo of components except for full exact match, - # then search the db for those combos - - allele_id_components = allele_id.split(",") - if all(component.startswith("PA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where( - VariantTranslation.aa_clingen_id == component - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == component, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[0]) - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[1]) - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - elif all(component.startswith("CA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where( - VariantTranslation.nt_clingen_id == allele_id - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[0]) - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[1]) - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - else: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}': all components must be either PA or CA", - ) - - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - return [variants_by_allele_id[allele_id] for allele_id in request.clingen_allele_ids] + +@router.get( + "/variants/{urn}/va/study-result", + status_code=200, + response_model=ExperimentalVariantFunctionalImpactStudyResult, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec StudyResult for a variant", +) +def get_variant_study_result( + *, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> JSONResponse: + """Construct a single VA-Spec StudyResult for a variant by URN, from its mapping substrate.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, detail=f"No study result exists for variant {urn}: no mapping data exists." + ) + + try: + return _annotation_response(variant_study_result(context), as_of) + except MappingDataDoesntExistException as e: + logger.info(msg=f"Could not construct a study result for variant {urn}: {e}", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"No study result exists for variant {urn}: {e}") @router.get( - "/variants/{urn}", + "/variants/{urn}/va/functional-statement", status_code=200, - response_model=VariantEffectMeasurementWithScoreSet, + response_model=Statement, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - response_model_exclude_none=True, - summary="Fetch variant by URN", + summary="Construct a VA-Spec functional-impact Statement for a variant", ) -def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user)): - """ - Fetch a single variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) +def get_variant_functional_impact_statement( + *, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> JSONResponse: + """Construct a single VA-Spec functional-impact Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No functional impact statement exists for variant {urn}: no mapping data exists.", + ) + try: - query = db.query(Variant).filter(Variant.urn == urn) - variant = query.one_or_none() - except MultipleResultsFound: - logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) - raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + functional_impact = variant_functional_impact_statement(context, principal=principal) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a functional impact statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No functional impact statement exists for variant {urn}: {e}") + + if not functional_impact: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its functional impact.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient " + "evidence to evaluate its functional impact." + ), + ) - if not variant: - logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) - raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + return _annotation_response(functional_impact, as_of) - assert_permission(user_data, variant.score_set, Action.READ) - return variant + +@router.get( + "/variants/{urn}/va/pathogenicity-statement", + status_code=200, + response_model=VariantPathogenicityStatement, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec pathogenicity Statement for a variant", +) +def get_variant_pathogenicity_statement( + *, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> JSONResponse: + """Construct a single VA-Spec pathogenicity Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No pathogenicity statement exists for variant {urn}: no mapping data exists.", + ) + + try: + pathogenicity_statement = variant_pathogenicity_statement(context, principal=principal) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a pathogenicity statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for variant {urn}: {e}") + + if not pathogenicity_statement: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its pathogenicity.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient " + "evidence to evaluate its pathogenicity." + ), + ) + + return _annotation_response(pathogenicity_statement, as_of) @router.get( @@ -480,9 +380,17 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User def get_variant_csv_namespaces( *, urn: str, + response: Response, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the offered namespaces as they stood at this instant, so discovery matches an " + "`as_of` download. ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), ) -> Any: """ List the CSV column namespaces this variant has data for, labeled and grouped for a picker. @@ -513,11 +421,14 @@ def get_variant_csv_namespaces( assert_permission(user_data, variant.score_set, Action.READ) + # Echoed like the CSV endpoints: the instant discovery answered for is part of the answer. + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" return available_variant_csv_namespaces( db, urn, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, viewer=principal.viewer_for(ScoreCalibrationViewer), + as_of=as_of, ) @@ -545,6 +456,14 @@ def get_variant_csv_data( db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), principal: Principal = Depends(get_principal), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the mapping-derived columns (reference HGVS, VEP, gnomAD, ClinVar) as they " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Defaults to current." + ), + ), ) -> Any: """ Return tabular data for a single variant, identified by URN, in CSV format. @@ -598,9 +517,14 @@ def get_variant_csv_data( namespaces=namespaces, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, viewer=principal.viewer_for(ScoreCalibrationViewer), + as_of=as_of, ) return StreamingResponse( iter([csv_str]), media_type="text/csv", - headers={"Content-Disposition": f'attachment; filename="{urn}.csv"'}, + headers={ + "Content-Disposition": f'attachment; filename="{urn}.csv"', + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "Access-Control-Expose-Headers": "X-As-Of", + }, ) diff --git a/src/mavedb/scripts/audit_allele_identifiers.py b/src/mavedb/scripts/audit_allele_identifiers.py new file mode 100644 index 000000000..fd0d1128d --- /dev/null +++ b/src/mavedb/scripts/audit_allele_identifiers.py @@ -0,0 +1,316 @@ +""" +Audit stored allele identifiers against their own content. + +Usage: +``` +python3 -m mavedb.scripts.audit_allele_identifiers audit [SCOPE] +python3 -m mavedb.scripts.audit_allele_identifiers audit-pre-mapped [SCOPE] +python3 -m mavedb.scripts.audit_allele_identifiers audit-locations [SCOPE] +``` + +SCOPE restricts each command to alleles and mapping records of some score sets; the options are the ones in +`mavedb.scripts.audit_scope`. Unscoped, each command covers the whole database. + +Every row in `alleles` asserts that `vrs_digest` is the GA4GH computed identifier of `post_mapped`. +Nothing enforced that until the mapping ingest path began recomputing identity on arrival, so rows +written earlier can carry an identifier minted before their own normalization. `audit` reports them, and `--repair` corrects the ones that can be corrected safely. + +Run this before and after +`alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py`. That migration now +recomputes identity as it writes, so it will not import drift — but `vrs_digest` is a UNIQUE dedup key +on an immutable, ValidTime-less table, so a wrong value there cannot be corrected in place afterwards. +Measuring before and expecting zero after is how that is kept honest. + +`--repair` renames only: it rewrites `vrs_digest` where the corrected identifier is unused. Where the +corrected identifier already exists, the row is a duplicate of one that was always right, and merging +it means repointing five tables (`annotation_events`, `clinvar_allele_links`, `gnomad_allele_links`, +`mapping_record_alleles`, `vep_allele_consequences`), four of which carry a unique index including +`allele_id` and can therefore collide. That is deliberately not automated. A duplicate appearing now +means a writer got past the ingest guard in `worker/jobs/variant_processing/mapping.py`, and that wants +investigating rather than papering over. + +`audit-pre-mapped` covers `mapping_records.vrs_digest` against `pre_mapped`. Not a dedup key, so a wrong +value is less destructive, but it is the same identity claim with the same cause and the backfill +republishes it. + +`audit-locations` covers the same defect one level down, on the nested `SequenceLocation.id`, which +`audit` does not look at and which no repair here can fix. The two are separate because the causes and +the remedies are separate — see that command. + +Read-only by default. Safe to run against production. +""" + +import logging +from collections import Counter +from datetime import datetime +from typing import Any, Optional, Sequence + +from ga4gh.core import ga4gh_identify +from ga4gh.vrs.models import Allele as VrsAllele +from ga4gh.vrs.models import CisPhasedBlock, SequenceLocation + +import asyncclick as click +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.lib.vrs_utils import identify_variation +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.scripts.audit_scope import resolve_scope, scope_options +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + +# Rows are streamed in batches of this size; buffering a production-sized result set exhausts memory. +STREAM_BATCH_SIZE = 5000 + + +def recomputed_identifier(post_mapped: dict[str, Any]) -> Optional[str]: + """The identifier this document's own content implies, or ``None`` if it cannot be hydrated. + + Hydration goes through ``vrs_object_from_mapped_variant`` rather than the strict VRS models + because older rows carry fields the current models forbid (``type`` on ``Extension``, ``label`` on + ``SequenceReference``). Neither affects the digest — it is computed over coordinates and state — so + the shim's tolerance costs nothing here. + """ + try: + variation = vrs_object_from_mapped_variant(dict(post_mapped)).root + # vrs_object_from_mapped_variant only ever produces one of these two (its own contract), but its + # declared return type is the full VRS MolecularVariation union, wider than identify_variation + # accepts. + if not isinstance(variation, (VrsAllele, CisPhasedBlock)): + return None + return identify_variation(variation) + except Exception: + logger.debug("Could not hydrate an allele for identification", exc_info=True) + return None + + +def _alleles_in_scope(score_set_ids: Optional[Sequence[int]]): + """Alleles with a ``post_mapped`` body, restricted to ones linked from the given score sets.""" + query = select(Allele).where(Allele.post_mapped.isnot(None)).order_by(Allele.id) + if score_set_ids is not None: + linked = select(MappingRecordAllele.allele_id).where(MappingRecordAllele.score_set_id.in_(score_set_ids)) + query = query.where(Allele.id.in_(linked)) + return query + + +@script_environment.command() +@scope_options +@click.option("--limit", type=int, default=None, help="Examine at most this many alleles.") +@click.option("--repair", is_flag=True, help="Rewrite vrs_digest where the corrected identifier is unused.") +@with_database_session +def audit( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + limit: Optional[int], + repair: bool, +) -> None: + """Report — and with ``--repair``, rename — alleles whose identifier disagrees with their content.""" + query = _alleles_in_scope(resolve_scope(db, urns, collection, translated_since)) + if limit is not None: + query = query.limit(limit) + + known = {digest for (digest,) in db.execute(select(Allele.vrs_digest)).all()} + counts: Counter[str] = Counter() + renames: list[tuple[Allele, str]] = [] + duplicates: list[tuple[Allele, str]] = [] + + for allele in db.scalars(query.execution_options(yield_per=STREAM_BATCH_SIZE)): + counts["examined"] += 1 + # post_mapped is nullable on the model, but the query above only selects rows where it isn't. + assert allele.post_mapped is not None + computed = recomputed_identifier(allele.post_mapped) + + if computed is None: + counts["could not hydrate"] += 1 + continue + if computed == allele.vrs_digest: + counts["identifier matches content"] += 1 + continue + + counts["DRIFTED"] += 1 + # A corrected identifier that already exists means the allele is a duplicate of a row that was + # always right: the drifted key simply failed to dedup against it. Those merge; the rest rename. + (duplicates if computed in known else renames).append((allele, computed)) + + for label, count in counts.most_common(): + logger.info("%8d %s", count, label) + logger.info("%8d drifted, corrected identifier is unused (rename)", len(renames)) + logger.info("%8d drifted, corrected identifier already exists (duplicate)", len(duplicates)) + + if repair and renames: + _rename(db, renames) + elif renames: + logger.warning("Run with --repair to correct %d rename(s).", len(renames)) + + if duplicates: + logger.error( + "%d allele(s) duplicate a row that was always right. Not repaired here — merging repoints " + "five tables, four with a unique index on allele_id. Investigate how they were written.", + len(duplicates), + ) + for allele, computed in duplicates[:20]: + logger.error(" allele id=%s stored=%s content=%s", allele.id, allele.vrs_digest, computed) + + +def _rename(db: Session, renames: list[tuple["Allele", str]]) -> None: + """Rewrite ``vrs_digest`` in place for rows whose corrected identifier is unused. + + Safe because the corrected identifier is known unused at classification time and this runs in one + transaction: a concurrent writer taking it in between surfaces as a ``uq_alleles_vrs_digest`` + violation on flush rather than a silent overwrite. Only the key changes — ``post_mapped`` is left + exactly as stored, since the body was never in question and allele bodies are immutable. + """ + for allele, computed in renames: + logger.info("renaming allele id=%s %s -> %s", allele.id, allele.vrs_digest, computed) + allele.vrs_digest = computed + + db.flush() + logger.info("renamed %d allele(s)", len(renames)) + + +@script_environment.command() +@scope_options +@click.option("--limit", type=int, default=None, help="Examine at most this many mapping records.") +@with_database_session +def audit_pre_mapped( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + limit: Optional[int], +) -> None: + """Report mapping records whose ``vrs_digest`` disagrees with their ``pre_mapped`` content. + + Read-only, and no repair: ``mapping_records.vrs_digest`` is indexed but not unique, so a wrong value + does not collapse two records into one the way it does on ``alleles``. It is still an identity claim + about content the backfill republishes, and it has the same cause — an id minted before VRS + normalization moved a del/dup span. The pre-mapped side had never been measured before 2026-08-25. + """ + query = select(MappingRecord).where(MappingRecord.pre_mapped.isnot(None)).order_by(MappingRecord.id) + if (score_set_ids := resolve_scope(db, urns, collection, translated_since)) is not None: + query = query.where(MappingRecord.score_set_id.in_(score_set_ids)) + if limit is not None: + query = query.limit(limit) + + counts: Counter[str] = Counter() + drifted: list[tuple[int, Optional[str], str]] = [] + + for record in db.scalars(query.execution_options(yield_per=STREAM_BATCH_SIZE)): + counts["examined"] += 1 + # pre_mapped is nullable on the model, but the query above only selects rows where it isn't. + assert record.pre_mapped is not None + computed = recomputed_identifier(record.pre_mapped) + + if computed is None: + counts["could not hydrate"] += 1 + elif computed == record.vrs_digest: + counts["identifier matches content"] += 1 + elif record.vrs_digest is None: + counts["no stored identifier"] += 1 + else: + counts["DRIFTED"] += 1 + drifted.append((record.id, record.vrs_digest, computed)) + + for label, count in counts.most_common(): + logger.info("%8d %s", count, label) + + for record_id, stored, computed in drifted[:20]: + logger.warning(" mapping_record id=%s stored=%s content=%s", record_id, stored, computed) + if len(drifted) > 20: + logger.warning(" ... and %d more", len(drifted) - 20) + + +@script_environment.command() +@scope_options +@click.option("--limit", type=int, default=None, help="Examine at most this many alleles.") +@with_database_session +def audit_locations( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + limit: Optional[int], +) -> None: + """Report nested ``SequenceLocation`` identifiers that disagree with their own coordinates. + + Separate from :func:`audit` because it is a separate defect with a separate cause. ``audit`` checks + ``vrs_digest`` against ``post_mapped`` — the allele's own identifier. ``ga4gh_identify`` writes only + the ``id`` of the object it is handed; for sub-objects it calls ``get_or_create_digest`` and never + ``get_or_create_ga4gh_identifier``. So an allele can carry a correct identifier over a location whose + ``id`` was minted before normalization moved the span, and the allele-level audit reports it clean. + + Read-only. A drifted location is repaired by re-mapping the score set, since both writers now + restamp the location — see ``lib/vrs_utils.py::identify_allele``. + """ + query = _alleles_in_scope(resolve_scope(db, urns, collection, translated_since)) + if limit is not None: + query = query.limit(limit) + + counts: Counter[str] = Counter() + drifted: set[int] = set() + + for allele in db.scalars(query.execution_options(yield_per=STREAM_BATCH_SIZE)): + counts["alleles examined"] += 1 + for path, location in _stored_locations(allele.post_mapped): + stored = location.get("id") + computed = _recomputed_location_identifier(location) + + if computed is None: + counts["could not hydrate location"] += 1 + elif stored is None: + counts["location carries no id"] += 1 + elif stored == computed: + counts["location id matches content"] += 1 + else: + counts["location id DRIFTED"] += 1 + drifted.add(allele.id) + logger.debug("Allele %s %s: %s -> %s", allele.id, path, stored, computed) + + for label, count in counts.most_common(): + logger.info("%9d %s", count, label) + logger.info("%9d distinct allele rows carrying a drifted location", len(drifted)) + + if drifted: + logger.warning( + "Re-map the affected score sets. A drifted location id is not repairable in place here — the " + "allele's own identifier is unaffected, so nothing in this audit's scope is wrong." + ) + + +def _stored_locations(post_mapped: Any) -> list[tuple[str, dict[str, Any]]]: + """Every ``SequenceLocation`` in a stored document, with a label saying where it sat. + + A ``CisPhasedBlock`` embeds one allele per member, each with its own location, and those member + copies are how one writer's convention leaked into another's rows. + """ + if not isinstance(post_mapped, dict): + return [] + + locations = [] + if isinstance(post_mapped.get("location"), dict): + locations.append(("location", post_mapped["location"])) + for index, member in enumerate(post_mapped.get("members") or []): + if isinstance(member, dict) and isinstance(member.get("location"), dict): + locations.append((f"members[{index}].location", member["location"])) + + return locations + + +def _recomputed_location_identifier(location: dict[str, Any]) -> Optional[str]: + """The identifier this location's own coordinates imply, ignoring what it currently claims.""" + try: + content = {key: value for key, value in location.items() if key not in ("id", "digest")} + return ga4gh_identify(SequenceLocation(**content), in_place="always") + except Exception: + logger.debug("Could not hydrate a location for identification", exc_info=True) + return None + + +if __name__ == "__main__": + script_environment() diff --git a/src/mavedb/scripts/audit_reverse_translation.py b/src/mavedb/scripts/audit_reverse_translation.py new file mode 100644 index 000000000..b40006273 --- /dev/null +++ b/src/mavedb/scripts/audit_reverse_translation.py @@ -0,0 +1,825 @@ +""" +Audit the allele graph that mapping and reverse translation (RT) write. + +Usage: +``` +python3 -m mavedb.scripts.audit_reverse_translation invariants [SCOPE] [--skip-seqrepo] +python3 -m mavedb.scripts.audit_reverse_translation outcomes [SCOPE] [--max-failed-pct 5] +python3 -m mavedb.scripts.audit_reverse_translation round-trip [SCOPE] [--per-score-set 50] +``` + +SCOPE is any of `--score-set URN` (repeatable), `--collection URN` and `--translated-since YYYY-MM-DD`, which +intersect. `--translated-since` keeps only score sets every variant of which RT reached on or after the date, so +findings left by an older pipeline drop out. Unscoped, every check covers the whole database. + +Each check guards a property the serving layer assumes but no constraint enforces. The class of bug +this exists for is the NM_007294.3 refget divergence: the mapper and RT built the same variant on two +sequences, both digests were internally consistent, every existing check passed, and convergent alleles +silently read as projections. + +`invariants` runs the structural checks in SQL, plus a comparison of every stored refget against +SeqRepo and a scan of SeqRepo for sequences carrying a ga4gh alias other than their own digest. `outcomes` rolls the latest RT result per variant up by score set, so an outlier score set +stands out. `round-trip` re-translates a sample of projection pairs with the same coordinate mapper RT +uses and checks that each pair and its protein apex identify as the same VRS alleles RT stored. + +Each finding has a severity. `error` is a broken invariant; `warning` is a shape that is suspicious but +has legitimate causes, so read the samples; `info` is a count for context. `invariants` and `round-trip` +exit with status 1 when any error-level finding is non-zero. + +Read-only. Safe to run against production, though `round-trip` makes one data-provider call per +translation and is paced by it. +""" + +import json +import logging +import re +import sqlite3 +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Callable, Optional, Sequence + +import asyncclick as click +from sqlalchemy import text +from sqlalchemy.orm import Session + +from mavedb.lib.hgvs import strip_protein_prediction_parens +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget +from mavedb.lib.vrs_utils import CANONICAL_ACCESSION_PREFIXES, identify_variation, translate_hgvs_to_variation +from mavedb.scripts.audit_scope import resolve_scope, scope_options +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + +ERROR = "error" +WARNING = "warning" +INFO = "info" + +REFGET_PATH = "{location,sequenceReference,refgetAccession}" + + +@dataclass +class Finding: + """The result of one check: how many rows violate it, and a few of them to read.""" + + name: str + severity: str + description: str + count: int + samples: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(frozen=True) +class SqlCheck: + """A check expressed as a query whose rows are the violations. + + ``sql`` may contain ``{allele_filter}``, ``{record_filter}`` or ``{link_filter}``, which expand to a + score-set restriction when the audit is scoped and to nothing otherwise. ``scoped=False`` marks a + check that has no meaning inside one score set (orphaned alleles belong to none) and is skipped then. + """ + + name: str + severity: str + description: str + sql: str + scoped: bool = True + + +# An allele some live mapping record links to. +_ALLELE_IS_LIVE = "EXISTS (SELECT 1 FROM mapping_record_alleles ll WHERE ll.allele_id = a.id AND ll.valid_to IS NULL)" + +# An allele whose state is longer or shorter than the span it replaces. An allele with a non-numeric +# (imprecise) or unreadable location counts as length-preserving, so it stays under the error-level check. +# CASE, not AND, guards the casts: Postgres does not promise to evaluate AND operands in order. +_LENGTH_CHANGING = """coalesce( + CASE WHEN jsonb_typeof(a.post_mapped #> '{location,start}') = 'number' + AND jsonb_typeof(a.post_mapped #> '{location,end}') = 'number' + THEN (a.post_mapped #>> '{location,end}')::int - (a.post_mapped #>> '{location,start}')::int + <> coalesce(length(a.post_mapped #>> '{state,sequence}'), (a.post_mapped #>> '{state,length}')::int) + END, + false +)""" + +# The live candidates RT linked to the measured allele's record, on its own level and sequence, span this +# interval. RT's candidates for one protein change all sit inside the codon it changes. +_CANDIDATE_SPAN_SQL = """ + LEFT JOIN LATERAL ( + SELECT min(CASE WHEN jsonb_typeof(da.post_mapped #> '{location,start}') = 'number' + THEN (da.post_mapped #>> '{location,start}')::int END) AS candidate_start, + max(CASE WHEN jsonb_typeof(da.post_mapped #> '{location,end}') = 'number' + THEN (da.post_mapped #>> '{location,end}')::int END) AS candidate_end + FROM mapping_record_alleles d + JOIN alleles da ON da.id = d.allele_id + WHERE d.mapping_record_id = auth.mapping_record_id AND NOT d.is_authoritative AND d.valid_to IS NULL + AND da.level = a.level + AND da.post_mapped #>> '{location,sequenceReference,refgetAccession}' + = a.post_mapped #>> '{location,sequenceReference,refgetAccession}' + ) span ON true +""" + +# A reference-identical allele (g.=, c.=). RT lists the alternative codons for a synonymous change, never the +# reference codon itself, so such an allele cannot fold in. +_REFERENCE_IDENTICAL = "coalesce(a.hgvs_g, a.hgvs_c) LIKE '%='" + +# A measured allele reaching past every candidate's span changes a base outside the codon RT targeted. With no +# candidate on its level and sequence, or an unreadable location, it counts as inside. +_BEYOND_CANDIDATES = """coalesce( + CASE WHEN jsonb_typeof(a.post_mapped #> '{location,start}') = 'number' + AND jsonb_typeof(a.post_mapped #> '{location,end}') = 'number' + THEN (a.post_mapped #>> '{location,start}')::int < span.candidate_start + OR (a.post_mapped #>> '{location,end}')::int > span.candidate_end + END, + false +)""" + +_MEASURED_OUTSIDE_GROUP_SQL = """ + SELECT auth.mapping_record_id, auth.allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c) AS hgvs + FROM mapping_record_alleles auth + JOIN alleles a ON a.id = auth.allele_id + {candidate_span} + WHERE auth.is_authoritative AND auth.valid_to IS NULL + AND NOT EXISTS (SELECT 1 FROM mapping_record_alleles pl + JOIN alleles pa ON pa.id = pl.allele_id + WHERE pl.projection_group = auth.projection_group AND pl.allele_id <> auth.allele_id + AND pl.valid_to IS NULL AND pa.level IN ('cdna', 'genomic')) + AND a.level IN ('cdna', 'genomic') + AND {condition} + {auth_filter} + AND EXISTS (SELECT 1 FROM mapping_record_alleles d + JOIN alleles da ON da.id = d.allele_id + WHERE d.mapping_record_id = auth.mapping_record_id + AND NOT d.is_authoritative AND d.valid_to IS NULL + AND da.level IN ('cdna', 'genomic')) +""".replace("{candidate_span}", _CANDIDATE_SPAN_SQL) + + +def _measured_outside_group(condition: str) -> str: + """The measured-allele-outside-projection-group query, restricted to alleles matching ``condition``.""" + return _MEASURED_OUTSIDE_GROUP_SQL.replace("{condition}", condition) + + +SQL_CHECKS: tuple[SqlCheck, ...] = ( + SqlCheck( + name="accession_with_multiple_refgets", + severity=ERROR, + description="Alleles on one accession carry more than one refget, so the same variant can mint two digests.", + sql=f""" + SELECT split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + array_agg(DISTINCT a.post_mapped #>> '{REFGET_PATH}') AS refgets, + count(*) AS alleles + FROM alleles a + WHERE a.post_mapped #>> '{REFGET_PATH}' IS NOT NULL {{allele_filter}} + GROUP BY 1 + HAVING count(DISTINCT a.post_mapped #>> '{REFGET_PATH}') > 1 + """, + ), + SqlCheck( + name="level_disagrees_with_hgvs", + severity=ERROR, + description="An allele's level disagrees with the HGVS column it fills or that expression's type.", + sql=""" + SELECT a.id AS allele_id, a.level, a.hgvs_g, a.hgvs_c, a.hgvs_p + FROM alleles a + WHERE (num_nonnulls(a.hgvs_g, a.hgvs_c, a.hgvs_p) <> 1 + OR NOT ((a.level = 'genomic' AND split_part(coalesce(a.hgvs_g, ''), ':', 2) LIKE 'g.%') + OR (a.level = 'cdna' AND split_part(coalesce(a.hgvs_c, ''), ':', 2) ~ '^[cn][.]') + OR (a.level = 'protein' AND split_part(coalesce(a.hgvs_p, ''), ':', 2) LIKE 'p.%'))) + {allele_filter} + """, + ), + SqlCheck( + name="accession_disagrees_with_level", + severity=WARNING, + description="An allele's accession is not the kind its level implies (NC_ genomic, NM_/ENST cdna, NP_/ENSP protein).", + sql=""" + SELECT a.id AS allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) AS hgvs + FROM alleles a + WHERE NOT ((a.level = 'genomic' AND coalesce(a.hgvs_g, '') ~ '^(NC|NG|NT|NW)_') + OR (a.level = 'cdna' AND coalesce(a.hgvs_c, '') ~ '^(NM_|NR_|XM_|XR_|ENST)') + OR (a.level = 'protein' AND coalesce(a.hgvs_p, '') ~ '^(NP_|XP_|ENSP)')) + {allele_filter} + """, + ), + SqlCheck( + name="clingen_allele_id_on_twin_alleles", + severity=ERROR, + description=( + "One ClinGen allele id sits on more than one live allele on the same accession, so two writers built one " + "change differently. A CA/PA id spans reference sequences, so the same id across accessions is expected." + ), + sql=f""" + SELECT a.clingen_allele_id, split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + count(*) AS alleles, array_agg(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) ORDER BY a.id) AS hgvs + FROM alleles a + WHERE a.clingen_allele_id IS NOT NULL AND {_ALLELE_IS_LIVE} {{allele_filter}} + GROUP BY 1, 2 + HAVING count(*) > 1 + """, + ), + SqlCheck( + name="clingen_allele_id_on_retired_twin", + severity=INFO, + description=( + "A ClinGen allele id shared with an allele no live record links to. Re-registration only submits live " + "alleles, so a superseded allele keeps the id it had, including a legacy id copied by variant URN." + ), + sql=f""" + SELECT a.clingen_allele_id, split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + count(*) AS alleles, array_agg(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) ORDER BY a.id) AS hgvs + FROM alleles a + WHERE a.clingen_allele_id IS NOT NULL {{allele_filter}} + GROUP BY 1, 2 + HAVING count(*) > 1 AND count(*) FILTER (WHERE {_ALLELE_IS_LIVE}) <= 1 + """, + ), + SqlCheck( + name="mapped_record_without_authoritative_link", + severity=ERROR, + description="A live mapping record whose latest mapping succeeded has no live authoritative allele.", + sql=""" + SELECT mr.id AS mapping_record_id, mr.variant_id, mr.score_set_id + FROM mapping_records mr + WHERE mr.valid_to IS NULL {record_filter} + AND NOT EXISTS (SELECT 1 FROM mapping_record_alleles l + WHERE l.mapping_record_id = mr.id AND l.is_authoritative AND l.valid_to IS NULL) + AND (SELECT e.disposition FROM annotation_event e + WHERE e.variant_id = mr.variant_id AND e.annotation_type = 'vrs_mapping' + ORDER BY e.id DESC LIMIT 1) = 'present' + """, + ), + SqlCheck( + name="live_link_on_retired_record", + severity=ERROR, + description="An allele link is live but its mapping record was retired; retiring a record retires its links.", + sql=""" + SELECT l.id AS link_id, l.mapping_record_id, l.allele_id, mr.valid_to AS record_retired_at + FROM mapping_record_alleles l + JOIN mapping_records mr ON mr.id = l.mapping_record_id + WHERE l.valid_to IS NULL AND mr.valid_to IS NOT NULL {link_filter} + """, + ), + SqlCheck( + name="measured_allele_outside_projection_group", + severity=ERROR, + description=( + "RT attached derived nucleotide alleles to a record, but the measured nucleotide allele has no partner " + "allele in its measured projection group: RT matched none of its candidates to it, so serving labels " + "every sibling convergent, its own c/g twin included. Only length-preserving changes inside the codon RT targeted, which its candidates " + "always include." + ), + sql=_measured_outside_group( + f"NOT {_LENGTH_CHANGING} AND NOT {_REFERENCE_IDENTICAL} AND NOT {_BEYOND_CANDIDATES}" + ), + ), + SqlCheck( + name="measured_allele_beyond_rt_codon", + severity=INFO, + description=( + "A measured change with no partner allele that reaches outside the codon RT targeted, such as a " + "multi-base change straddling two codons. RT lists only single-codon changes, so it cannot fold in. " + "Many in one score set can mean its nucleotide variants sit out of frame with its protein variants." + ), + sql=_measured_outside_group(f"NOT {_LENGTH_CHANGING} AND NOT {_REFERENCE_IDENTICAL} AND {_BEYOND_CANDIDATES}"), + ), + SqlCheck( + name="measured_reference_allele_outside_projection_group", + severity=INFO, + description=( + "A measured reference-identical allele (g.=, c.=) with no partner allele. Expected: RT lists the " + "alternative codons for a synonymous change, never the reference codon." + ), + sql=_measured_outside_group(_REFERENCE_IDENTICAL), + ), + SqlCheck( + name="measured_indel_outside_projection_group", + severity=INFO, + description=( + "A measured indel with no partner allele. Expected when the indel is not codon-aligned: it changes the " + "protein the same way as a codon-aligned candidate but is a different DNA change, so RT does not list it." + ), + sql=_measured_outside_group(f"{_LENGTH_CHANGING} AND NOT {_REFERENCE_IDENTICAL}"), + ), + SqlCheck( + name="malformed_projection_group", + severity=WARNING, + description=( + "A live candidate projection group is not exactly one cdna allele and one genomic allele. Either the library could " + "not project the coding candidate to the genome (its entry under candidates in the variant's " + "cross_level_translation event has no hgvs_g), or a member failed VRS translation (listed under " + "failed_candidates with the error)." + ), + sql=""" + SELECT l.mapping_record_id, l.projection_group, count(*) AS members, + string_agg(a.level, ',' ORDER BY a.level) AS levels + FROM mapping_record_alleles l + JOIN alleles a ON a.id = l.allele_id + WHERE l.valid_to IS NULL AND l.projection_group IS NOT NULL {link_filter} + -- Measured groups legitimately lack an allele at a level VRS can't represent (intronic). + AND NOT EXISTS (SELECT 1 FROM mapping_record_alleles auth + WHERE auth.projection_group = l.projection_group + AND auth.is_authoritative AND auth.valid_to IS NULL) + GROUP BY 1, 2 + HAVING string_agg(a.level, ',' ORDER BY a.level) <> 'cdna,genomic' + """, + ), + SqlCheck( + name="record_with_multiple_protein_alleles", + severity=WARNING, + description="A live mapping record links more than one protein allele; RT shares a single protein apex per record.", + sql=""" + SELECT l.mapping_record_id, count(*) AS protein_alleles, + array_agg(coalesce(a.hgvs_p, a.vrs_digest) ORDER BY a.id) AS hgvs + FROM mapping_record_alleles l + JOIN alleles a ON a.id = l.allele_id + WHERE l.valid_to IS NULL AND a.level = 'protein' {link_filter} + GROUP BY 1 + HAVING count(*) > 1 + """, + ), + SqlCheck( + name="mapped_record_never_reverse_translated", + severity=INFO, + description="A live record with an authoritative allele has no RT event; expected until RT has run on its score set.", + sql=""" + SELECT mr.id AS mapping_record_id, mr.variant_id, mr.score_set_id + FROM mapping_records mr + WHERE mr.valid_to IS NULL {record_filter} + AND EXISTS (SELECT 1 FROM mapping_record_alleles l + WHERE l.mapping_record_id = mr.id AND l.is_authoritative AND l.valid_to IS NULL) + AND NOT EXISTS (SELECT 1 FROM annotation_event e + WHERE e.variant_id = mr.variant_id AND e.annotation_type = 'cross_level_translation') + """, + ), + SqlCheck( + name="allele_without_live_link", + severity=INFO, + description=( + "An allele no live mapping record links to. Superseded mappings, including every non-current legacy " + "row the backfill imported, leave alleles with retired links only; one with no link at all was never linked." + ), + sql=""" + SELECT a.id AS allele_id, a.level, coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p) AS hgvs + FROM alleles a + WHERE NOT EXISTS (SELECT 1 FROM mapping_record_alleles l WHERE l.allele_id = a.id AND l.valid_to IS NULL) + """, + scoped=False, + ), +) + + +def _filters(scoped: bool) -> dict[str, str]: + """Score-set restrictions for each alias a check's SQL can use, or empty strings when unscoped.""" + if not scoped: + return {"allele_filter": "", "record_filter": "", "link_filter": "", "auth_filter": ""} + + return { + "allele_filter": ( + "AND EXISTS (SELECT 1 FROM mapping_record_alleles sl " + "WHERE sl.allele_id = a.id AND sl.score_set_id = ANY(:score_set_ids))" + ), + "record_filter": "AND mr.score_set_id = ANY(:score_set_ids)", + "link_filter": "AND l.score_set_id = ANY(:score_set_ids)", + "auth_filter": "AND auth.score_set_id = ANY(:score_set_ids)", + } + + +def run_sql_check(db: Session, check: SqlCheck, score_set_ids: Optional[list[int]], samples: int) -> Finding: + """Count a check's violations and fetch the first few, optionally within some score sets.""" + # str.replace rather than str.format: the SQL holds JSON paths whose braces format would consume. + sql = check.sql + for placeholder, fragment in _filters(score_set_ids is not None).items(): + sql = sql.replace(f"{{{placeholder}}}", fragment) + params: dict[str, Any] = {"samples": samples} + if score_set_ids is not None: + params["score_set_ids"] = score_set_ids + + count = db.execute(text(f"SELECT count(*) FROM ({sql}) violations"), params).scalar_one() + rows = ( + db.execute(text(f"SELECT * FROM ({sql}) violations LIMIT :samples"), params).mappings().all() if count else [] + ) + return Finding(check.name, check.severity, check.description, count, [dict(row) for row in rows]) + + +def check_refgets_against_seqrepo( + db: Session, + resolve: Callable[[str], str], + score_set_ids: Optional[list[int]], + samples: int, +) -> list[Finding]: + """Compare every stored (accession, refget) pair with the refget SeqRepo holds for the accession. + + ``accession_with_multiple_refgets`` misses an accession whose alleles all agree on the wrong + sequence; this catches it. Only accessions with a canonical prefix are checked, matching the + insert-time guard in :func:`mavedb.lib.vrs_utils.verify_allele_refget`. + + ``resolve`` is :func:`mavedb.lib.seqrepo.resolve_refget` bound to a SeqRepo; it is a parameter so tests + need not open one. + """ + pairs_sql = f""" + SELECT split_part(coalesce(a.hgvs_g, a.hgvs_c, a.hgvs_p), ':', 1) AS accession, + a.post_mapped #>> '{REFGET_PATH}' AS refget, + count(*) AS alleles + FROM alleles a + WHERE a.post_mapped #>> '{REFGET_PATH}' IS NOT NULL {_filters(score_set_ids is not None)["allele_filter"]} + GROUP BY 1, 2 + """ + params = {"score_set_ids": score_set_ids} if score_set_ids is not None else {} + + mismatched: list[dict[str, Any]] = [] + unresolved: list[dict[str, Any]] = [] + expected_by_accession: dict[str, Optional[str]] = {} + + for accession, refget, alleles in db.execute(text(pairs_sql), params).all(): + if not accession.startswith(CANONICAL_ACCESSION_PREFIXES): + continue + + if accession not in expected_by_accession: + try: + expected_by_accession[accession] = resolve(accession) + except (SequenceNotFoundError, AmbiguousSequenceError) as e: + expected_by_accession[accession] = None + unresolved.append({"accession": accession, "error": str(e)}) + + expected = expected_by_accession[accession] + if expected is not None and refget != expected: + mismatched.append({"accession": accession, "stored": refget, "seqrepo": expected, "alleles": alleles}) + + return [ + Finding( + "refget_disagrees_with_seqrepo", + ERROR, + "Alleles sit on a refget other than the one SeqRepo holds for their accession.", + sum(row["alleles"] for row in mismatched), + mismatched[:samples], + ), + Finding( + "accession_unresolved_in_seqrepo", + WARNING, + "SeqRepo has no sequence, or more than one, for an accession alleles use.", + len(unresolved), + unresolved[:samples], + ), + ] + + +# SeqRepo stores each ga4gh alias in its VMC namespace as ``GS_``. A sequence's only correct ga4gh +# alias is its own digest; any other is current and makes refget lookups return either digest. +FOREIGN_REFGET_ALIAS_SQL = """ + SELECT seqalias_id, seq_id, alias, added + FROM seqalias + WHERE namespace = 'VMC' AND is_current = 1 AND alias != 'GS_' || seq_id + ORDER BY added +""" + + +def check_foreign_refget_aliases(aliases_db: sqlite3.Connection, samples: int) -> Finding: + """Find SeqRepo sequences carrying a ga4gh alias other than their own digest. + + Every lookup that turns an accession into a refget takes the first ga4gh alias of the sequence, so a + second one makes the answer vary between processes. Not scoped: SeqRepo is shared by every score set. + """ + rows = [ + {"seqalias_id": seqalias_id, "seq_id": seq_id, "alias": alias, "added": added} + for seqalias_id, seq_id, alias, added in aliases_db.execute(FOREIGN_REFGET_ALIAS_SQL) + ] + return Finding( + "seqrepo_foreign_refget_alias", + ERROR, + "A SeqRepo sequence carries a ga4gh alias that is not its own digest, so refget lookups for it are " + "nondeterministic. Retire the alias (is_current = 0) and re-map the score sets whose records use it.", + len(rows), + rows[:samples], + ) + + +def report(findings: Sequence[Finding]) -> int: + """Log each finding with its samples and return how many error-level findings are non-zero.""" + for finding in findings: + status = "ok" if finding.count == 0 else finding.severity.upper() + logger.info("%-8s %10d %s", status, finding.count, finding.name) + if finding.count: + logger.info(" %s", finding.description) + for sample in finding.samples: + logger.info(" %s", json.dumps(sample, default=str)) + + return sum(1 for finding in findings if finding.severity == ERROR and finding.count) + + +@script_environment.command() +@scope_options +@click.option("--samples", type=int, default=5, show_default=True, help="Violating rows to print per check.") +@click.option("--skip-seqrepo", is_flag=True, help="Skip the refget comparison, which needs the SeqRepo mount.") +@with_database_session +def invariants( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + samples: int, + skip_seqrepo: bool, +) -> None: + """Check the structural invariants of the mapping and reverse-translation allele graph.""" + score_set_ids = resolve_scope(db, urns, collection, translated_since) + + findings: list[Finding] = [] + for check in SQL_CHECKS: + if score_set_ids is not None and not check.scoped: + continue + logger.info("running %s", check.name) + findings.append(run_sql_check(db, check, score_set_ids, samples)) + + if not skip_seqrepo: + from mavedb.data_providers.services import seqrepo + + sr = seqrepo() + logger.info("running refget_disagrees_with_seqrepo") + findings.extend(check_refgets_against_seqrepo(db, lambda acc: resolve_refget(sr, acc), score_set_ids, samples)) + logger.info("running seqrepo_foreign_refget_alias") + findings.append(check_foreign_refget_aliases(sr.aliases._db, samples)) + + if errors := report(findings): + logger.error("%d error-level check(s) failed.", errors) + sys.exit(1) + + +RT_OUTCOMES_SQL = """ + WITH latest AS ( + SELECT DISTINCT ON (e.variant_id) e.variant_id, e.score_set_id, e.disposition, e.reason + FROM annotation_event e + WHERE e.annotation_type = 'cross_level_translation' {event_filter} + ORDER BY e.variant_id, e.id DESC + ) + SELECT s.urn, latest.disposition, latest.reason, count(*) AS variants + FROM latest + JOIN scoresets s ON s.id = latest.score_set_id + GROUP BY 1, 2, 3 +""" + + +@dataclass +class ScoreSetOutcome: + """The latest RT result of every variant in one score set, by disposition and by failure reason.""" + + urn: str + by_disposition: Counter = field(default_factory=Counter) + failure_reasons: Counter = field(default_factory=Counter) + + @property + def total(self) -> int: + return sum(self.by_disposition.values()) + + @property + def failed_pct(self) -> float: + return 100.0 * self.by_disposition["failed"] / self.total if self.total else 0.0 + + +def collect_outcomes(rows: Sequence[tuple[str, str, str, int]]) -> list[ScoreSetOutcome]: + """Fold ``(urn, disposition, reason, variants)`` rows into one outcome per score set, worst first.""" + outcomes: dict[str, ScoreSetOutcome] = {} + for urn, disposition, reason, variants in rows: + outcome = outcomes.setdefault(urn, ScoreSetOutcome(urn)) + outcome.by_disposition[disposition] += variants + if disposition == "failed": + outcome.failure_reasons[reason] += variants + + return sorted(outcomes.values(), key=lambda o: (-o.failed_pct, o.urn)) + + +@script_environment.command() +@scope_options +@click.option( + "--max-failed-pct", + type=float, + default=5.0, + show_default=True, + help="Flag score sets where more than this share of variants failed RT.", +) +@with_database_session +def outcomes( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + max_failed_pct: float, +) -> None: + """Summarize the latest reverse-translation result per variant, by score set.""" + score_set_ids = resolve_scope(db, urns, collection, translated_since) + event_filter = "AND e.score_set_id = ANY(:score_set_ids)" if score_set_ids is not None else "" + params = {"score_set_ids": score_set_ids} if score_set_ids is not None else {} + + rows = db.execute(text(RT_OUTCOMES_SQL.format(event_filter=event_filter)), params).all() + results = collect_outcomes([tuple(row) for row in rows]) # type: ignore[misc] + + logger.info("%-28s %9s %9s %9s %9s %s", "score set", "variants", "present", "n/a", "failed %", "failure reasons") + for outcome in results: + flag = " <-- above threshold" if outcome.failed_pct > max_failed_pct else "" + reasons = ", ".join(f"{reason} {n}" for reason, n in outcome.failure_reasons.most_common(3)) + logger.info( + "%-28s %9d %9d %9d %8.1f%% %s%s", + outcome.urn, + outcome.total, + outcome.by_disposition["present"], + outcome.by_disposition["not_applicable"], + outcome.failed_pct, + reasons, + flag, + ) + + flagged = [o.urn for o in results if o.failed_pct > max_failed_pct] + logger.info( + "%d score set(s) reverse translated; %d above %.1f%% failed.", len(results), len(flagged), max_failed_pct + ) + + +ROUND_TRIP_SQL = """ + WITH pairs AS ( + SELECT l.score_set_id, l.mapping_record_id, l.projection_group, + max(a.hgvs_c) FILTER (WHERE a.level = 'cdna') AS hgvs_c, + max(a.vrs_digest) FILTER (WHERE a.level = 'cdna') AS cdna_digest, + max(a.hgvs_g) FILTER (WHERE a.level = 'genomic') AS hgvs_g, + max(a.vrs_digest) FILTER (WHERE a.level = 'genomic') AS genomic_digest + FROM mapping_record_alleles l + JOIN alleles a ON a.id = l.allele_id + WHERE l.valid_to IS NULL AND l.projection_group IS NOT NULL {link_filter} + GROUP BY 1, 2, 3 + HAVING count(*) FILTER (WHERE a.level = 'cdna') = 1 AND count(*) FILTER (WHERE a.level = 'genomic') = 1 + ), + sampled AS ( + SELECT pairs.*, + row_number() OVER (PARTITION BY score_set_id ORDER BY mapping_record_id, projection_group) AS n + FROM pairs + ) + SELECT sampled.mapping_record_id, sampled.projection_group, sampled.hgvs_c, sampled.cdna_digest, + sampled.hgvs_g, sampled.genomic_digest, + apex.digests AS protein_digests, apex.hgvs AS protein_hgvs + FROM sampled + LEFT JOIN LATERAL ( + SELECT array_agg(pa.vrs_digest ORDER BY pa.id) AS digests, array_agg(pa.hgvs_p ORDER BY pa.id) AS hgvs + FROM mapping_record_alleles pl + JOIN alleles pa ON pa.id = pl.allele_id + WHERE pl.mapping_record_id = sampled.mapping_record_id AND pl.valid_to IS NULL AND pa.level = 'protein' + ) apex ON true + WHERE sampled.n <= :per_score_set + ORDER BY sampled.mapping_record_id, sampled.projection_group +""" + + +_UNCHANGED_PROTEIN_RE = re.compile(r":p\.(=|[A-Z][a-z]{2}\d+=|Ter\d+Ter|\*\d+\*)$") + + +def _is_unchanged_protein(hgvs_p: str) -> bool: + """Whether a protein HGVS expression says the protein sequence is unchanged.""" + return bool(_UNCHANGED_PROTEIN_RE.search(hgvs_p)) + + +@dataclass(frozen=True) +class ProjectionPair: + """One live projection group as stored, plus the protein alleles live on its record.""" + + mapping_record_id: int + projection_group: int + hgvs_c: str + cdna_digest: str + hgvs_g: str + genomic_digest: str + protein_digests: Sequence[str] + protein_hgvs: Sequence[Optional[str]] + + +def round_trip_pair( + pair: ProjectionPair, + c_to_g: Callable[[str], str], + c_to_p: Callable[[str], str], + digest_of: Callable[[str], str], +) -> list[tuple[str, dict[str, Any]]]: + """Re-derive a pair's genomic member and protein apex from its cdna member and compare digests. + + Comparing digests rather than strings makes the check indifferent to how an expression is written + (3' shifting, prediction parentheses): two expressions for the same change identify to one VRS + allele. Returns ``(outcome, detail)`` tuples, one for the genomic side and one for the protein side. + + The callables are RT's coordinate mapper and HGVS-to-digest translation; parameters so tests can + supply fakes instead of a data provider and SeqRepo. + """ + results: list[tuple[str, dict[str, Any]]] = [] + base = { + "mapping_record_id": pair.mapping_record_id, + "projection_group": pair.projection_group, + "hgvs_c": pair.hgvs_c, + } + + try: + derived_g = c_to_g(pair.hgvs_c) + derived_g_digest = digest_of(derived_g) + except Exception as e: + results.append(("genomic: could not re-derive", {**base, "error": str(e)})) + else: + if derived_g_digest == pair.genomic_digest: + results.append(("genomic: matches", base)) + else: + results.append( + ( + "GENOMIC: DISAGREES", + {**base, "stored": pair.hgvs_g, "derived": derived_g, "derived_digest": derived_g_digest}, + ) + ) + + digests = [d for d in pair.protein_digests if d is not None] + if not digests: + results.append(("protein: no apex on record", base)) + return results + + try: + derived_p = strip_protein_prediction_parens(c_to_p(pair.hgvs_c)) + # A start-codon change has no predictable protein effect (p.? or p.Met1?), and VRS cannot express one. + if derived_p.endswith("?"): + results.append(("protein: unknown effect (start codon)", {**base, "derived": derived_p})) + return results + derived_p_digest = digest_of(derived_p) + except Exception as e: + results.append(("protein: could not re-derive", {**base, "error": str(e)})) + return results + + if derived_p_digest in digests: + results.append(("protein: matches", base)) + elif _is_unchanged_protein(derived_p) and all(_is_unchanged_protein(h) for h in pair.protein_hgvs if h): + # Different notations for the same unchanged protein: hgvs writes an insertion that creates a stop right + # after the last residue as p.Ter330Ter, while RT's apex is p.Thr328=. Their digests differ. + results.append(("protein: matches (unchanged)", base)) + else: + results.append( + ( + "PROTEIN: DISAGREES", + {**base, "stored": list(pair.protein_hgvs), "derived": derived_p, "derived_digest": derived_p_digest}, + ) + ) + + return results + + +@script_environment.command() +@scope_options +@click.option( + "--per-score-set", + type=int, + default=50, + show_default=True, + help="Projection pairs to re-translate in each score set.", +) +@click.option("--samples", type=int, default=10, show_default=True, help="Rows to print per disagreeing outcome.") +@with_database_session +def round_trip( + db: Session, + urns: tuple[str, ...], + collection: Optional[str], + translated_since: Optional[datetime], + per_score_set: int, + samples: int, +) -> None: + """Re-translate sampled projection pairs and check they identify as the alleles RT stored.""" + from ga4gh.vrs.extras.translator import AlleleTranslator + + from mavedb.data_providers.services import cdot_rest, seqrepo_data_proxy + from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + + score_set_ids = resolve_scope(db, urns, collection, translated_since) + link_filter = "AND l.score_set_id = ANY(:score_set_ids)" if score_set_ids is not None else "" + params: dict[str, Any] = {"per_score_set": per_score_set} + if score_set_ids is not None: + params["score_set_ids"] = score_set_ids + + coordinates = WorkerCoordinateTranslator(cdot_rest()) + translator = AlleleTranslator(seqrepo_data_proxy()) + + def digest_of(hgvs: str) -> str: + return identify_variation(translate_hgvs_to_variation(hgvs, translator)) + + counts: Counter[str] = Counter() + examples: dict[str, list[dict[str, Any]]] = defaultdict(list) + rows = db.execute(text(ROUND_TRIP_SQL.format(link_filter=link_filter)), params).mappings().all() + logger.info("re-translating %d projection pair(s)", len(rows)) + + for row in rows: + pair = ProjectionPair( + **{**row, "protein_digests": row["protein_digests"] or [], "protein_hgvs": row["protein_hgvs"] or []} + ) + for outcome, detail in round_trip_pair(pair, coordinates.c_to_g, coordinates.c_to_p, digest_of): + counts[outcome] += 1 + if len(examples[outcome]) < samples: + examples[outcome].append(detail) + + for outcome, count in sorted(counts.items()): + logger.info("%10d %s", count, outcome) + if not outcome.endswith("matches"): + for detail in examples[outcome]: + logger.info(" %s", json.dumps(detail, default=str)) + + disagreements = counts["GENOMIC: DISAGREES"] + counts["PROTEIN: DISAGREES"] + if disagreements: + logger.error("%d re-translation(s) disagree with the stored alleles.", disagreements) + sys.exit(1) + + +if __name__ == "__main__": + script_environment() diff --git a/src/mavedb/scripts/audit_scope.py b/src/mavedb/scripts/audit_scope.py new file mode 100644 index 000000000..2f7fd24ac --- /dev/null +++ b/src/mavedb/scripts/audit_scope.py @@ -0,0 +1,85 @@ +""" +Score-set scoping shared by the audit scripts. + +An unscoped audit covers the whole database, so on an environment where only some score sets have been +through the current pipeline, findings from stale score sets mix with real ones. These options restrict an +audit to the score sets that matter. Given together, they intersect. +""" + +import datetime +from typing import Callable, Optional, Sequence + +import asyncclick as click +from sqlalchemy import select, text +from sqlalchemy.orm import Session + +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.score_set import ScoreSet + +# A score set every one of whose variants has an RT event on or after :since. RT writes an event for every +# variant it processes, not_applicable and failed included, so a variant RT has not reached since then keeps +# the score set out. +TRANSLATED_SINCE_SQL = """ + SELECT score_set_id + FROM ( + SELECT e.score_set_id, e.variant_id, max(e.created_at) AS translated_at + FROM annotation_event e + WHERE e.annotation_type = 'cross_level_translation' + GROUP BY 1, 2 + ) latest + GROUP BY score_set_id + HAVING min(translated_at) >= :since +""" + + +def scope_options(command: Callable) -> Callable: + """Add ``--score-set``, ``--collection`` and ``--translated-since`` to an audit command.""" + command = click.option( + "--translated-since", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Only score sets whose every variant was reverse translated on or after this date (UTC).", + )(command) + command = click.option("--collection", default=None, help="Only score sets in this collection (URN).")(command) + command = click.option("--score-set", "urns", multiple=True, help="Only this score set (URN). Repeatable.")(command) + return command + + +def resolve_scope( + db: Session, + urns: Sequence[str], + collection: Optional[str], + translated_since: Optional[datetime.datetime], +) -> Optional[list[int]]: + """The score-set ids the scope options select, or ``None`` when none were given.""" + if not urns and collection is None and translated_since is None: + return None + + query = select(ScoreSet.id) + + if urns: + found = set(db.scalars(select(ScoreSet.urn).where(ScoreSet.urn.in_(urns)))) + if missing := sorted(set(urns) - found): + raise click.BadParameter(f"No score set with URN: {', '.join(missing)}", param_hint="--score-set") + query = query.where(ScoreSet.urn.in_(urns)) + + if collection is not None: + if db.scalar(select(Collection.id).where(Collection.urn == collection)) is None: + raise click.BadParameter(f"No collection with URN: {collection}", param_hint="--collection") + query = ( + query.join(CollectionScoreSetAssociation, CollectionScoreSetAssociation.score_set_id == ScoreSet.id) + .join(Collection, Collection.id == CollectionScoreSetAssociation.collection_id) + .where(Collection.urn == collection) + ) + + if translated_since is not None: + since = translated_since.replace(tzinfo=datetime.timezone.utc) + translated = [row[0] for row in db.execute(text(TRANSLATED_SINCE_SQL), {"since": since})] + query = query.where(ScoreSet.id.in_(translated)) + + score_set_ids = list(db.scalars(query)) + if not score_set_ids: + raise click.UsageError("No score set matches the given scope.") + + return score_set_ids diff --git a/src/mavedb/scripts/backfill_campaign.py b/src/mavedb/scripts/backfill_campaign.py new file mode 100644 index 000000000..61f8ba8be --- /dev/null +++ b/src/mavedb/scripts/backfill_campaign.py @@ -0,0 +1,799 @@ +"""Read-only campaign map and safe-to-drop gate for the MappedVariant → allele-substrate backfill. + +The backfill has two halves (see ``alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py`` +for the reshape and ``run_score_set_pipelines.py`` for the enrichment): a deterministic *reshape* that +reconstructs the measured-level spine and source-annotation links from the frozen ``mapped_variants`` rows, +and a decoupled *enrichment* that runs reverse translation (the cross-level fan-out) and re-runs the source +annotations against the substrate. This script does not move data; it reports where every published score set +sits between those two states, and whether the frozen ``mapped_variants`` tables are safe to drop. + +Commands +-------- +coverage + The campaign's map of what's left. For each score set: reshaped? RT fan-out present? gnomAD / ClinVar / + ClinGen / VEP enrichment present (and, when versions are supplied, at the current version)? Roll-up + counts show how many score sets are in each state. + +reconcile + The gate that should block dropping the ``mapped_variants`` tables. Reports measured-level coverage + parity between the frozen tables and the substrate, the expected divergence the substrate adds (RT + fan-out, advanced annotation versions), and a static scan confirming no code outside the known + migration/export allowlist still queries the legacy tables. Exits non-zero unless the drop is safe, so it + can gate a release step in CI or a runbook. + +Usage: + poetry run python -m mavedb.scripts.backfill_campaign coverage --published-only + poetry run python -m mavedb.scripts.backfill_campaign coverage --published-only --incomplete-only --json + poetry run python -m mavedb.scripts.backfill_campaign reconcile --published-only + poetry run python -m mavedb.scripts.backfill_campaign reconcile --published-only \\ + --vep-source-version 116 --vep-resolver-version 3 +""" + +import dataclasses +import json +import logging +import pathlib +import re +from typing import Optional, Sequence + +import asyncclick as click +from sqlalchemy import and_, distinct, func, select +from sqlalchemy.orm import Session + +from mavedb.db.session import SessionLocal +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.scripts.environment import script_environment + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Coverage model +# --------------------------------------------------------------------------- + + +# The lifecycle states a score set moves through during the campaign, in order. classify_state returns the +# furthest state a score set has reached; the roll-up counts how many sit at each. +STATE_NONE = "none" # no live mapping_record: reshape has not run +STATE_PARTIAL_RESHAPE = "partial_reshape" # some but not all legacy-current variants have a live record +STATE_RESHAPED = "reshaped" # measured-level spine complete, no RT fan-out yet +STATE_RT = "rt" # reverse-translation fan-out present, but enrichment incomplete +STATE_ENRICHED = "enriched" # RT + gnomAD + ClinVar + ClinGen + VEP all present (at current version if pinned) + +STATE_ORDER = [STATE_NONE, STATE_PARTIAL_RESHAPE, STATE_RESHAPED, STATE_RT, STATE_ENRICHED] + + +@dataclasses.dataclass +class ScoreSetCoverage: + """Substrate state of one score set, as counts over its variants and their live alleles.""" + + score_set_id: int + urn: Optional[str] + n_variants: int = 0 + n_legacy_variants: int = 0 # variants with a live (current) legacy MappedVariant — the reshape denominator + n_reshaped_variants: int = 0 # variants with a live MappingRecord + n_live_alleles: int = 0 + n_nonauthoritative_alleles: int = 0 # RT fan-out breadth; reshape mints only authoritative alleles + has_projection_group: bool = False + n_clingen_alleles: int = 0 + n_gnomad_alleles: int = 0 + n_clinvar_alleles: int = 0 + n_vep_alleles: int = 0 + n_vep_current_alleles: int = 0 # only meaningful when VEP versions were pinned on the command line + + @property + def is_reshaped(self) -> bool: + """Every legacy-served variant has a live MappingRecord (the measured level is complete).""" + return self.n_legacy_variants > 0 and self.n_reshaped_variants >= self.n_legacy_variants + + @property + def is_partially_reshaped(self) -> bool: + return 0 < self.n_reshaped_variants < self.n_legacy_variants + + @property + def has_rt(self) -> bool: + """Reverse translation ran: it is the only producer of non-authoritative sibling alleles.""" + return self.n_nonauthoritative_alleles > 0 + + @property + def has_gnomad(self) -> bool: + return self.n_gnomad_alleles > 0 + + @property + def has_clinvar(self) -> bool: + return self.n_clinvar_alleles > 0 + + @property + def has_clingen(self) -> bool: + return self.n_clingen_alleles > 0 + + def has_vep(self, *, require_current: bool) -> bool: + return (self.n_vep_current_alleles if require_current else self.n_vep_alleles) > 0 + + +def classify_state(cov: ScoreSetCoverage, *, require_current_vep: bool) -> str: + """The furthest lifecycle state a score set has reached. + + ``enriched`` is defined by having all four source annotations present — NOT by the RT fan-out breadth. + Reverse translation legitimately produces zero sibling alleles for a single-level score set (nothing to + project across coordinate frames), so gating ``enriched`` on non-authoritative alleles would strand such a + score set at ``reshaped`` forever even after it is fully annotated. VEP depends on reverse translation + (SUCCESS_REQUIRED), so a live VEP consequence already implies RT ran; the fan-out count is reported as its + own column rather than used as a state gate. + + A score set legitimately carrying no ClinVar or gnomAD overlap would never reach ``enriched`` by the + presence rule; the campaign treats presence as the signal and relies on the reconcile parity report, not + this roll-up, to catch a score set that *should* have had a link and doesn't. + """ + if cov.n_reshaped_variants == 0: + return STATE_NONE + if cov.is_partially_reshaped: + return STATE_PARTIAL_RESHAPE + fully_enriched = ( + cov.has_gnomad and cov.has_clinvar and cov.has_clingen and cov.has_vep(require_current=require_current_vep) + ) + if fully_enriched: + return STATE_ENRICHED + # RT has run (fan-out siblings exist, or a VEP consequence — which depends on RT — is present) but at least + # one source annotation is still missing. + if cov.has_rt or cov.has_vep(require_current=False): + return STATE_RT + return STATE_RESHAPED + + +# --------------------------------------------------------------------------- +# Coverage queries +# --------------------------------------------------------------------------- + + +def resolve_cohort_ids( + db: Session, + *, + published_only: bool, + include_private: bool, + collection_urn: Optional[str], + explicit_urns: Optional[Sequence[str]], +) -> list[int]: + """Score-set ids in scope. Filters AND together; ``--published-only`` covers accession-based score sets + identically to sequence-based ones (it discriminates only on ``published_date``), so the historical + taxonomy-join pitfall that silently dropped accession-based score sets cannot recur here.""" + query = select(ScoreSet.id).where(ScoreSet.urn.is_not(None)) + if published_only: + query = query.where(ScoreSet.published_date.is_not(None)) + if not include_private and not published_only: + query = query.where(ScoreSet.private.is_(False)) + if explicit_urns: + query = query.where(ScoreSet.urn.in_(list(explicit_urns))) + if collection_urn: + from mavedb.models.collection import Collection + from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation + + query = ( + query.join(CollectionScoreSetAssociation, CollectionScoreSetAssociation.score_set_id == ScoreSet.id) + .join(Collection, Collection.id == CollectionScoreSetAssociation.collection_id) + .where(Collection.urn == collection_urn) + ) + return list(db.scalars(query).all()) + + +def compute_coverage( + db: Session, + score_set_ids: Sequence[int], + *, + vep_source_version: Optional[str] = None, + vep_resolver_version: Optional[str] = None, +) -> dict[int, ScoreSetCoverage]: + """Substrate coverage for each score set id, as a handful of grouped aggregate passes joined in Python. + + Every count is over *live* substrate rows (``valid_to IS NULL``); a superseded record or link does not + count as coverage. Allele-level counts run over the live authoritative-and-RT alleles reachable from each + score set's live mapping records. + """ + ids = list(score_set_ids) + coverage = {sid: ScoreSetCoverage(score_set_id=sid, urn=None) for sid in ids} + if not ids: + return coverage + + for sid, urn in db.execute(select(ScoreSet.id, ScoreSet.urn).where(ScoreSet.id.in_(ids))): + coverage[sid].urn = urn + + for sid, n in db.execute( + select(Variant.score_set_id, func.count(Variant.id)) + .where(Variant.score_set_id.in_(ids)) + .group_by(Variant.score_set_id) + ): + coverage[sid].n_variants = n + + for sid, n in db.execute( + select(Variant.score_set_id, func.count(distinct(MappedVariant.variant_id))) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(Variant.score_set_id.in_(ids), MappedVariant.current.is_(True)) + .group_by(Variant.score_set_id) + ): + coverage[sid].n_legacy_variants = n + + for sid, n in db.execute( + select(Variant.score_set_id, func.count(distinct(MappingRecord.variant_id))) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id.in_(ids), MappingRecord.valid_to.is_(None)) + .group_by(Variant.score_set_id) + ): + coverage[sid].n_reshaped_variants = n + + # Allele-level aggregates over live records -> live record-allele links -> alleles. + live_allele_join = ( + select( + Variant.score_set_id.label("sid"), + func.count(distinct(MappingRecordAllele.allele_id)).label("n_alleles"), + func.count(distinct(MappingRecordAllele.allele_id)) + .filter(MappingRecordAllele.is_authoritative.is_(False)) + .label("n_nonauth"), + func.bool_or(MappingRecordAllele.projection_group.is_not(None)).label("has_pg"), + func.count(distinct(MappingRecordAllele.allele_id)) + .filter(Allele.clingen_allele_id.is_not(None)) + .label("n_clingen"), + ) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + ), + ) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id.in_(ids)) + .group_by(Variant.score_set_id) + ) + for sid, n_alleles, n_nonauth, has_pg, n_clingen in db.execute(live_allele_join): + cov = coverage[sid] + cov.n_live_alleles = n_alleles + cov.n_nonauthoritative_alleles = n_nonauth + cov.has_projection_group = bool(has_pg) + cov.n_clingen_alleles = n_clingen + + _fill_link_counts(db, ids, coverage, GnomadAlleleLink, "n_gnomad_alleles") + _fill_link_counts(db, ids, coverage, ClinvarAlleleLink, "n_clinvar_alleles") + _fill_vep_counts(db, ids, coverage, vep_source_version, vep_resolver_version) + + return coverage + + +def _live_allele_ids_for_scoreset(): + """A correlated live-allele subquery keyed by score set, shared by the per-source link counters.""" + return ( + select(Variant.score_set_id.label("sid"), MappingRecordAllele.allele_id.label("allele_id")) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + ), + ) + ) + + +def _fill_link_counts(db: Session, ids: Sequence[int], coverage: dict, link_model, attr: str) -> None: + live_alleles = _live_allele_ids_for_scoreset().where(Variant.score_set_id.in_(ids)).subquery() + query = ( + select(live_alleles.c.sid, func.count(distinct(live_alleles.c.allele_id))) + .join(link_model, and_(link_model.allele_id == live_alleles.c.allele_id, link_model.valid_to.is_(None))) + .group_by(live_alleles.c.sid) + ) + for sid, n in db.execute(query): + setattr(coverage[sid], attr, n) + + +def _fill_vep_counts( + db: Session, + ids: Sequence[int], + coverage: dict, + vep_source_version: Optional[str], + vep_resolver_version: Optional[str], +) -> None: + live_alleles = _live_allele_ids_for_scoreset().where(Variant.score_set_id.in_(ids)).subquery() + base = ( + select(live_alleles.c.sid, func.count(distinct(live_alleles.c.allele_id))) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == live_alleles.c.allele_id, VepAlleleConsequence.valid_to.is_(None)), + ) + .group_by(live_alleles.c.sid) + ) + for sid, n in db.execute(base): + coverage[sid].n_vep_alleles = n + + if vep_source_version is None: + return + # "Current" mirrors the VEP job's own skip predicate: both the Ensembl release and the resolver version + # must match. A pinned resolver version of "" is treated as "match the source version only". + conds = [VepAlleleConsequence.source_version == vep_source_version] + if vep_resolver_version: + conds.append(VepAlleleConsequence.resolver_version == vep_resolver_version) + current = ( + select(live_alleles.c.sid, func.count(distinct(live_alleles.c.allele_id))) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == live_alleles.c.allele_id, VepAlleleConsequence.valid_to.is_(None)), + ) + .where(*conds) + .group_by(live_alleles.c.sid) + ) + for sid, n in db.execute(current): + coverage[sid].n_vep_current_alleles = n + + +# --------------------------------------------------------------------------- +# coverage command +# --------------------------------------------------------------------------- + + +def _coverage_row(cov: ScoreSetCoverage, state: str) -> dict: + return { + "score_set_urn": cov.urn or "(no urn)", + "state": state, + "n_variants": cov.n_variants, + "n_legacy": cov.n_legacy_variants, + "n_reshaped": cov.n_reshaped_variants, + "reshaped": cov.is_reshaped, + "rt": cov.has_rt, + "n_rt_alleles": cov.n_nonauthoritative_alleles, + "gnomad": cov.has_gnomad, + "clinvar": cov.has_clinvar, + "clingen": cov.has_clingen, + "vep": cov.has_vep(require_current=False), + "vep_current": cov.n_vep_current_alleles, + } + + +@script_environment.command(name="coverage") +@click.option("--published-only", is_flag=True, help="Only published score sets (covers accession-based sets).") +@click.option("--include-private", is_flag=True, help="Include private score sets (ignored with --published-only).") +@click.option("--collection-urn", default=None, help="Only score sets in this collection.") +@click.option("--score-set-urn", "score_set_urns", multiple=True, help="Restrict to these URNs (repeatable).") +@click.option("--vep-source-version", default=None, help="Ensembl release counted as current for VEP (e.g. 116).") +@click.option("--vep-resolver-version", default=None, help="Resolver version counted as current for VEP.") +@click.option("--incomplete-only", is_flag=True, help="Only score sets not yet fully enriched.") +@click.option("--limit", type=int, default=None, help="Cap rows (after filtering); the roll-up still counts all.") +@click.option("--json", "as_json", is_flag=True, help="Emit rows as JSON.") +async def coverage( + published_only: bool, + include_private: bool, + collection_urn: Optional[str], + score_set_urns: tuple[str, ...], + vep_source_version: Optional[str], + vep_resolver_version: Optional[str], + incomplete_only: bool, + limit: Optional[int], + as_json: bool, +) -> None: + """The campaign's map of what's left: substrate state of every in-scope score set.""" + require_current_vep = vep_source_version is not None + with SessionLocal() as db: + ids = resolve_cohort_ids( + db, + published_only=published_only, + include_private=include_private, + collection_urn=collection_urn, + explicit_urns=score_set_urns or None, + ) + coverage_by_id = compute_coverage( + db, ids, vep_source_version=vep_source_version, vep_resolver_version=vep_resolver_version + ) + + covs = sorted(coverage_by_id.values(), key=lambda c: c.urn or "") + states = {cov.score_set_id: classify_state(cov, require_current_vep=require_current_vep) for cov in covs} + + rollup = {state: 0 for state in STATE_ORDER} + for state in states.values(): + rollup[state] += 1 + + rows = [_coverage_row(cov, states[cov.score_set_id]) for cov in covs] + if incomplete_only: + rows = [r for r in rows if r["state"] != STATE_ENRICHED] + + if as_json: + click.echo( + json.dumps({"rollup": rollup, "total": len(covs), "score_sets": rows[:limit]}, indent=2, default=str) + ) + return + + click.echo(f"Cohort: {len(covs)} score set(s)") + click.echo("Campaign state roll-up (furthest state reached):") + for state in STATE_ORDER: + click.echo(f" {state:<16} {rollup[state]:>6}") + if require_current_vep: + click.echo(f" (VEP 'current' pinned to source={vep_source_version} resolver={vep_resolver_version or 'any'})") + click.echo() + + shown = rows[:limit] + if not shown: + click.echo("No score sets to show.") + return + + header = ( + f"{'URN':<26} {'STATE':<16} {'VARS':>7} {'LEGACY':>7} {'RESHAP':>7} " + f"{'RT':>3} {'RTALL':>6} {'GNO':>3} {'CLV':>3} {'CLG':>3} {'VEP':>3}" + ) + click.echo(header) + click.echo("-" * len(header)) + for r in shown: + click.echo( + f"{r['score_set_urn']:<26} {r['state']:<16} {r['n_variants']:>7} {r['n_legacy']:>7} " + f"{r['n_reshaped']:>7} {_yn(r['rt']):>3} {r['n_rt_alleles']:>6} {_yn(r['gnomad']):>3} " + f"{_yn(r['clinvar']):>3} {_yn(r['clingen']):>3} {_yn(r['vep']):>3}" + ) + if limit is not None and len(rows) > limit: + click.echo(f"... {len(rows) - limit} more row(s) not shown (--limit {limit}).") + + +def _yn(value: bool) -> str: + return "yes" if value else "-" + + +# --------------------------------------------------------------------------- +# reconcile command — the safe-to-drop gate +# --------------------------------------------------------------------------- + + +# Files allowed to query the legacy MappedVariant tables. The reshape migration reads them by design; the +# export script's legacy artifact and this campaign tool query them intentionally, and are retired/removed with +# the drop. Any other file issuing a query against the frozen tables would be stranded by the drop, so the +# reader scan fails on a query idiom found outside this set. +# +# Deliberately NOT in scope: ORM relationship declarations, imports, type aliases, and docstrings that merely +# name MappedVariant. Those are definitions, not readers — dropping the tables breaks them at mapper-config +# time, but they are removed as part of the drop changeset itself (the runbook enumerates them), not cleared +# ahead of it. The scan matches executable query idioms so those declarations do not produce noise. +LEGACY_READER_ALLOWLIST = { + "src/mavedb/scripts/export_public_data.py", + "src/mavedb/scripts/mapped_gene_from_mapped_variant.py", + "src/mavedb/scripts/backfill_campaign.py", +} + +# A reader is a SQLAlchemy query construct against the model or its association tables — a select/query +# targeting it, a join onto it, or an eager-loader pulling it. This intentionally does not match +# ``relationship(... "MappedVariant")``, ``secondary=..._association_table``, ``delete(MappedVariant)`` (a +# writer, removed with the drop), imports, ``Union[..., MappedVariant, ...]``, or prose. +_LEGACY_READER_PATTERNS = [ + re.compile(r"\bselect\(\s*MappedVariant\b"), + re.compile(r"\.query\(\s*MappedVariant\b"), + re.compile(r"\bjoin\(\s*MappedVariant\b"), + re.compile(r"\b(?:joinedload|selectinload|contains_eager|subqueryload)\([^)]*MappedVariant\b"), + # The association tables are only ever referenced as ``secondary=..._association_table`` in a relationship + # (a declaration, ignored) or in a real query, where a call paren sits immediately before the raw table + # name: ``select_from(gnomad_variants_mapped_variants)``, ``join(mapped_variants_clinical_controls, ...)``. + # The trailing ``\b`` before ``_association_table`` fails to match, so the ``secondary=`` symbol is skipped. + re.compile(r"\((?:gnomad_variants_mapped_variants|mapped_variants_clinical_controls)\b"), +] + + +@dataclasses.dataclass +class AnnotationParity: + """Per-variant annotation coverage for one source, legacy vs substrate. + + ``regressed`` is the load-bearing number: variants the frozen tables annotated for this source that have + NO live substrate annotation of it on any of their live alleles. Dropping the frozen tables while this is + non-zero silently regresses that source's serving. It is expected to be non-zero after reshape-only for a + protein-authoritative score set — the reshape force-closes gnomAD/ClinVar links on a protein allele, and + only the RT fan-out's genomic/coding sibling plus re-annotation makes them live again — which is exactly + why enrichment must complete before the drop. + """ + + source: str + legacy_variants: int + substrate_variants: int + regressed_variants: int + + +@dataclasses.dataclass +class ReconcileReport: + cohort_size: int + legacy_current_variants: int + reshaped_variants: int + missing_reshape_variants: int # legacy-current variants with no live MappingRecord — the coverage gate + rt_fanout_alleles: int # expected divergence: substrate breadth the legacy tables never had + annotation_parity: list[AnnotationParity] + unexpected_legacy_readers: list[str] + + @property + def measured_parity_ok(self) -> bool: + return self.missing_reshape_variants == 0 + + @property + def annotation_parity_ok(self) -> bool: + """No source lost live per-variant coverage the frozen tables had.""" + return all(parity.regressed_variants == 0 for parity in self.annotation_parity) + + @property + def readers_clear(self) -> bool: + return not self.unexpected_legacy_readers + + @property + def safe_to_drop(self) -> bool: + return self.measured_parity_ok and self.annotation_parity_ok and self.readers_clear + + +def scan_legacy_readers(repo_root: pathlib.Path) -> list[str]: + """Repo-relative ``path:line`` sites, outside the allowlist, that *query* the frozen MappedVariant tables. + + This is the static half of the drop gate: the reconcile DB parity proves the substrate *covers* the data, + and this proves nothing outside the migration/export allowlist still *reads* the frozen tables. A new query + reader added after the campaign lands trips this even if the data is fully backfilled. + """ + src = repo_root / "src" / "mavedb" + hits: list[str] = [] + for path in sorted(src.rglob("*.py")): + rel = path.relative_to(repo_root).as_posix() + if rel in LEGACY_READER_ALLOWLIST: + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + for lineno, line in enumerate(text.splitlines(), start=1): + if line.lstrip().startswith("#"): + continue + if any(pattern.search(line) for pattern in _LEGACY_READER_PATTERNS): + hits.append(f"{rel}:{lineno}") + return hits + + +def compute_reconcile(db: Session, score_set_ids: Sequence[int], *, repo_root: pathlib.Path) -> ReconcileReport: + ids = list(score_set_ids) + id_filter = Variant.score_set_id.in_(ids) if ids else None + + def scoped(query): + return query.where(id_filter) if id_filter is not None else query + + legacy_current_variants = db.scalar( + scoped( + select(func.count(distinct(MappedVariant.variant_id))) + .select_from(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(MappedVariant.current.is_(True)) + ) + ) + + reshaped_variants = db.scalar( + scoped( + select(func.count(distinct(MappingRecord.variant_id))) + .select_from(MappingRecord) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(MappingRecord.valid_to.is_(None)) + ) + ) + + # The gate: legacy-current variants with no live MappingRecord. Dropping the frozen tables while this is + # non-zero strands those variants — they would have no substrate row to serve from. + live_record_exists = ( + select(MappingRecord.id) + .where(MappingRecord.variant_id == MappedVariant.variant_id, MappingRecord.valid_to.is_(None)) + .exists() + ) + missing_reshape_variants = db.scalar( + scoped( + select(func.count(distinct(MappedVariant.variant_id))) + .select_from(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(MappedVariant.current.is_(True), ~live_record_exists) + ) + ) + + rt_fanout_alleles = db.scalar( + scoped( + select(func.count(distinct(MappingRecordAllele.allele_id))) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + MappingRecordAllele.is_authoritative.is_(False), + ), + ) + ) + ) + + annotation_parity = _annotation_parity(db, id_filter) + + return ReconcileReport( + cohort_size=len(ids), + legacy_current_variants=legacy_current_variants or 0, + reshaped_variants=reshaped_variants or 0, + missing_reshape_variants=missing_reshape_variants or 0, + rt_fanout_alleles=rt_fanout_alleles or 0, + annotation_parity=annotation_parity, + unexpected_legacy_readers=scan_legacy_readers(repo_root), + ) + + +def _substrate_link_exists(variant_id_col, link_model): + """A correlated EXISTS: the variant has a live *link_model* row on a live allele of its live record.""" + return ( + select(1) + .select_from(MappingRecord) + .join( + MappingRecordAllele, + and_(MappingRecordAllele.mapping_record_id == MappingRecord.id, MappingRecordAllele.valid_to.is_(None)), + ) + .join(link_model, and_(link_model.allele_id == MappingRecordAllele.allele_id, link_model.valid_to.is_(None))) + .where(MappingRecord.variant_id == variant_id_col, MappingRecord.valid_to.is_(None)) + .exists() + ) + + +def _annotation_parity(db: Session, id_filter) -> list["AnnotationParity"]: + """Per-variant gnomAD / ClinVar / VEP coverage, legacy vs substrate, with the regression that gates the drop. + + For each source: how many current-mapped variants the frozen tables annotate, how many the substrate + annotates live, and — the gate — how many the frozen tables annotate that the substrate does NOT annotate + live. A non-zero regression means dropping the frozen tables would lose that source's serving for those + variants, so it must reach zero (via enrichment) before the drop. + """ + from mavedb.models.clinical_control_mapped_variant import ( + mapped_variants_clinical_controls_association_table as clinvar_assoc, + ) + from mavedb.models.gnomad_variant_mapped_variant import ( + gnomad_variants_mapped_variants_association_table as gnomad_assoc, + ) + + def legacy_variant_ids(join_or_predicate): + query = ( + select(distinct(MappedVariant.variant_id).label("variant_id")) + .select_from(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .where(MappedVariant.current.is_(True)) + ) + query = join_or_predicate(query) + if id_filter is not None: + query = query.where(id_filter) + return query.subquery() + + def substrate_variant_count(link_model): + query = ( + select(func.count(distinct(Variant.id))) + .select_from(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) + .join( + MappingRecordAllele, + and_(MappingRecordAllele.mapping_record_id == MappingRecord.id, MappingRecordAllele.valid_to.is_(None)), + ) + .join( + link_model, and_(link_model.allele_id == MappingRecordAllele.allele_id, link_model.valid_to.is_(None)) + ) + ) + if id_filter is not None: + query = query.where(id_filter) + return db.scalar(query) or 0 + + def parity(source, legacy_ids_subq, link_model) -> AnnotationParity: + legacy = db.scalar(select(func.count()).select_from(legacy_ids_subq)) or 0 + regressed = ( + db.scalar( + select(func.count()) + .select_from(legacy_ids_subq) + .where(~_substrate_link_exists(legacy_ids_subq.c.variant_id, link_model)) + ) + or 0 + ) + return AnnotationParity( + source=source, + legacy_variants=legacy, + substrate_variants=substrate_variant_count(link_model), + regressed_variants=regressed, + ) + + return [ + parity( + "gnomad", + legacy_variant_ids(lambda q: q.join(gnomad_assoc, gnomad_assoc.c.mapped_variant_id == MappedVariant.id)), + GnomadAlleleLink, + ), + parity( + "clinvar", + legacy_variant_ids(lambda q: q.join(clinvar_assoc, clinvar_assoc.c.mapped_variant_id == MappedVariant.id)), + ClinvarAlleleLink, + ), + parity( + "vep", + legacy_variant_ids(lambda q: q.where(MappedVariant.vep_functional_consequence.is_not(None))), + VepAlleleConsequence, + ), + ] + + +@script_environment.command(name="reconcile") +@click.option("--published-only", is_flag=True, help="Only published score sets.") +@click.option("--include-private", is_flag=True, help="Include private score sets (ignored with --published-only).") +@click.option("--collection-urn", default=None, help="Only score sets in this collection.") +@click.option("--score-set-urn", "score_set_urns", multiple=True, help="Restrict to these URNs (repeatable).") +@click.option("--json", "as_json", is_flag=True, help="Emit the report as JSON.") +async def reconcile( + published_only: bool, + include_private: bool, + collection_urn: Optional[str], + score_set_urns: tuple[str, ...], + as_json: bool, +) -> None: + """Safe-to-drop gate: measured-level parity + expected divergence + a static legacy-reader scan. + + Exits non-zero unless the drop is safe (every legacy-served variant is reshaped and no code outside the + migration/export allowlist still reads the frozen tables), so it can gate a runbook or CI step. + """ + repo_root = pathlib.Path(__file__).resolve().parents[3] + with SessionLocal() as db: + ids = resolve_cohort_ids( + db, + published_only=published_only, + include_private=include_private, + collection_urn=collection_urn, + explicit_urns=score_set_urns or None, + ) + report = compute_reconcile(db, ids, repo_root=repo_root) + + if as_json: + click.echo( + json.dumps(dataclasses.asdict(report) | {"safe_to_drop": report.safe_to_drop}, indent=2, default=str) + ) + else: + _print_reconcile(report) + + raise SystemExit(0 if report.safe_to_drop else 1) + + +def _print_reconcile(report: ReconcileReport) -> None: + click.echo(f"Reconcile over {report.cohort_size or 'ALL'} score set(s)") + click.echo() + click.echo("Measured-level parity (frozen mapped_variants -> substrate):") + click.echo(f" legacy current variants {report.legacy_current_variants:>10}") + click.echo(f" reshaped variants (live record) {report.reshaped_variants:>10}") + click.echo(f" MISSING reshape (gate) {report.missing_reshape_variants:>10} (must be 0 to drop)") + click.echo() + click.echo(f"RT fan-out sibling alleles the substrate adds (legacy had none): {report.rt_fanout_alleles}") + click.echo() + click.echo("Per-variant annotation parity (legacy -> substrate live; REGRESSED gates the drop):") + click.echo(f" {'source':<8} {'legacy':>8} {'substrate':>9} {'REGRESSED':>9}") + for parity in report.annotation_parity: + click.echo( + f" {parity.source:<8} {parity.legacy_variants:>8} " + f"{parity.substrate_variants:>9} {parity.regressed_variants:>9}" + ) + click.echo(" (a variant is 'regressed' if the frozen tables annotate it and the live substrate does not;") + click.echo(" reshape force-closes gnomAD/ClinVar on protein alleles, so enrichment must clear this to 0)") + click.echo() + click.echo("Legacy-table readers outside the allowlist:") + if report.unexpected_legacy_readers: + for hit in report.unexpected_legacy_readers: + click.echo(f" ! {hit}") + else: + click.echo(" none") + click.echo() + verdict = "SAFE TO DROP" if report.safe_to_drop else "NOT SAFE TO DROP" + click.echo(f"==> {verdict}") + if not report.safe_to_drop: + if not report.measured_parity_ok: + click.echo(f" - {report.missing_reshape_variants} legacy-served variant(s) are not reshaped.") + if not report.annotation_parity_ok: + regressed = ", ".join( + f"{p.source}={p.regressed_variants}" for p in report.annotation_parity if p.regressed_variants + ) + click.echo(f" - annotation coverage regressed for: {regressed} (run enrichment).") + if not report.readers_clear: + click.echo(f" - {len(report.unexpected_legacy_readers)} unexpected legacy reader(s) remain.") + + +if __name__ == "__main__": + script_environment() diff --git a/src/mavedb/scripts/backfill_score_distributions.py b/src/mavedb/scripts/backfill_score_distributions.py new file mode 100644 index 000000000..09c5dd3cb --- /dev/null +++ b/src/mavedb/scripts/backfill_score_distributions.py @@ -0,0 +1,85 @@ +"""Fill ``ScoreSet.score_distribution`` for score sets whose variants predate the column. + +New uploads compute the summary in the variant creation job; this one-off covers existing score sets. +Score sets are processed one at a time so only one score set's variant data is in memory at once. With +``--commit``, each score set commits as it finishes, so a run against a serving database never holds row locks on +more than one score set at a time and an interrupted run keeps its progress. + +Usage: + + python -m mavedb.scripts.backfill_score_distributions --dry-run + python -m mavedb.scripts.backfill_score_distributions --commit + python -m mavedb.scripts.backfill_score_distributions --commit --force +""" + +import logging + +import asyncclick as click +from sqlalchemy import select, update +from sqlalchemy.orm import Session + +from mavedb.lib.score_distribution import summarize_variant_data +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.scripts.environment import DatabaseSessionAction, script_environment, with_database_session + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +VARIANT_FETCH_BATCH_SIZE = 10_000 + + +def backfill(db: Session, force: bool = False, commit_each: bool = False) -> tuple[int, int]: + """Summarize each score set with variants; returns ``(filled, without_numeric_scores)``. + + ``commit_each`` commits after every score set instead of leaving the transaction to the caller. + """ + query = select(ScoreSet.id).where(ScoreSet.num_variants > 0).order_by(ScoreSet.id) + if not force: + query = query.where(ScoreSet.score_distribution.is_(None)) + score_set_ids = db.scalars(query).all() + logger.info(f"Summarizing scores for {len(score_set_ids)} score sets.") + + filled = 0 + empty = 0 + for index, score_set_id in enumerate(score_set_ids, start=1): + variant_data = db.scalars( + select(Variant.data) + .where(Variant.score_set_id == score_set_id) + .execution_options(yield_per=VARIANT_FETCH_BATCH_SIZE) + ) + summary = summarize_variant_data(variant_data) + + # Setting modification_date to itself keeps its onupdate from stamping every score set as modified today. + db.execute( + update(ScoreSet) + .where(ScoreSet.id == score_set_id) + .values(score_distribution=summary, modification_date=ScoreSet.modification_date) + ) + if commit_each: + db.commit() + else: + db.flush() + + if summary is None: + empty += 1 + else: + filled += 1 + + if index % 100 == 0: + logger.info(f"Processed {index} of {len(score_set_ids)} score sets.") + + return filled, empty + + +@script_environment.command() +@with_database_session(pass_action=True) +@click.option("--force", is_flag=True, help="Recompute summaries that already exist.") +def backfill_score_distributions(db: Session, action: DatabaseSessionAction, force: bool) -> None: + """Compute the score distribution summary for every score set with variants.""" + filled, empty = backfill(db, force, commit_each=action is DatabaseSessionAction.COMMIT) + logger.info(f"Filled {filled} score distributions; {empty} score sets had no numeric scores.") + + +if __name__ == "__main__": + backfill_score_distributions() diff --git a/src/mavedb/scripts/calibrated_variant_effects.py b/src/mavedb/scripts/calibrated_variant_effects.py index a8f470886..d209b436d 100644 --- a/src/mavedb/scripts/calibrated_variant_effects.py +++ b/src/mavedb/scripts/calibrated_variant_effects.py @@ -74,7 +74,16 @@ @click.command() @with_database_session def main(db: Session) -> None: - """Count unique variant effect measurements with ACMG-classified functional ranges.""" + """Count unique variant effect measurements with ACMG-classified functional ranges. + + Counts every calibration on a public score set, applying no ``ScoreCalibrationViewer``. A calibration + keeps its own ``private`` flag and a READ rule stricter than its score set's, so some of what is + counted here is not readable by any caller the API would serve. That is deliberate: this is an + operator's census of what MaveDB holds, not a response to a request, and a viewer-filtered count would + under-report the corpus. It is also the aggregate blind spot ``lib/permissions/viewer.py`` names — a + count leaks no calibration's contents, but it does reveal that private ones exist. Keep the output + internal, and do not lift these totals into anything caller-facing without applying a viewer. + """ query = ( select(ScoreSet) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 1fc15c5cb..dd20f8fb1 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -11,6 +11,10 @@ Unpublished data and data sets licensed other than under the Creative Commons Zero license are not included in the dump, and user details are limited to ORCID IDs and names of contributors to published data sets. + +RELEASING A NEW VERSION: Before publishing a new version of this archive to Zenodo, add an entry describing what +changed to `src/mavedb/scripts/resources/CHANGELOG.md` (it is bundled into the archive). The full release procedure +is documented in `deployment/docs/zenodo-release.md`. """ import json @@ -26,6 +30,12 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation +from mavedb.lib.annotation.conformance import serialize_annotation +from mavedb.lib.annotation.context import load_annotation_context_inputs +from mavedb.lib.allele_annotations import get_allele_cross_references +from mavedb.lib.alleles import get_live_record_allele_links, with_chunk_inputs +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.lib.csv.entries import score_sets_have_current_mappings from mavedb.lib.csv.namespaces import CsvNamespace from mavedb.lib.csv.score_set import ( available_score_set_csv_namespaces, @@ -33,7 +43,7 @@ ) from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation +from mavedb.lib.score_sets import get_annotatable_variants, readable_score_set_urns from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License @@ -57,33 +67,29 @@ """The only license whose data the dump may carry.""" -def annotation_export_namespaces(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> list[str]: - """The namespaces the public annotations CSV should carry for this score set. +def _emit_legacy_mapped_variants_artifact() -> bool: + """Whether to emit the deprecated ``mapped/{urn}.mapped-variants.json`` artifact. - *viewer* has no default on purpose. Discovery resolves an omitted viewer to the public subset, which - is the right answer for the dump but the wrong way to arrive at it: the archive's audience is a - decision this script makes, so it says so rather than inheriting it. - - Asks discovery what the score set actually has rather than naming groups by hand. The previous - hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases - never ingested, needed a code change for every new release, and was fragile to schema changes. + Superseded by ``vrs/{urn}.vrs.ndjson`` and removed with the MappedVariant table drop; kept for a + deprecation window until then. Set ``EXPORT_LEGACY_MAPPED_VARIANTS=0`` (or ``false``/``no``/``off``) to + stop emitting it before the boundary. Defaults to on. + """ + return os.getenv("EXPORT_LEGACY_MAPPED_VARIANTS", "1").strip().lower() not in {"0", "false", "no", "off"} - The archive carries everything MaveDB holds about the score set, so this takes what discovery found - and subtracts from it rather than opting groups in. - In particular it does not filter on `selected_by_default`. That flag answers "what should a download - dialog open on", which is a question about attention rather than about what exists, and the reasons a - group opens unchecked are not interchangeable. An archive is about completeness, not about what a user - should be nudged to look at first. +def annotation_export_namespaces(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> list[str]: + """The namespaces the public annotations CSV should carry for this score set. - Nor does it filter on `research_use_only`. A research-use-only calibration is public data, and every - group it produces carries a `research_use_only` column stating its standing, so a consumer can filter - on the data itself. VA-Spec NDJSON follows a different rule currently, see TODO(#803). + *viewer* has no default: the dump always targets the anonymous/public audience, and that should be + explicit here rather than inherited from discovery's default. - Subtractions: + Derived from discovery rather than a hand-maintained list, so a new release needs no code change here. + Includes everything discovery finds regardless of `selected_by_default` (a UI attention flag, not a + completeness one) or `research_use_only` (each group carries its own `research_use_only` column, so + consumers can filter on the data itself — VA-Spec NDJSON follows a different rule, see TODO(#803)). - - Every score and count group, and the score set's own identity: scores and counts get their own - files, and the URN is in the filename, so repeating either would be noise. + Excludes the score, count, and score-set-identity groups: those get their own files, and the URN is + already in the filename. """ excluded = { CsvNamespace.SCORES, @@ -103,20 +109,31 @@ def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: def archive_path_base(score_set_urn: str) -> str: - """The filename stem a score set's artifacts share, e.g. ``urn-mavedb-00000001-a-1``. + """The filename stem shared by a score set's artifacts, e.g. ``urn-mavedb-00000001-a-1``. - Colons are not portable in archive member names on every platform, so the URN is hyphenated. The - README documents the substitution as the way back to the URN, which makes it part of the published - contract rather than an implementation detail. + Colons aren't portable in archive member names on every platform, so the URN is hyphenated. The + README documents the substitution as the way back to the URN. """ return score_set_urn.replace(":", "-") def score_set_has_current_mappings(db: Session, score_set: ScoreSet) -> bool: - """Whether any variant in the score set has a current mapping. + """Whether any variant in the score set has a live mapping on the allele substrate. + + Gates the artifacts built from that substrate, so a score set whose mappings are all superseded + doesn't emit empty files. Reads the same predicate CSV discovery uses rather than the frozen + ``MappedVariant`` table, which no longer gets writes and would report every score set mapped since + the migration as unmapped. + """ + # TODO(#372): non-null id fields + return score_sets_have_current_mappings(db, [score_set.id]) # type: ignore - Gates the three mapping-derived artifacts. A score set whose mappings are all superseded yields no - annotations, so emitting empty files for it would advertise absence as data. + +def score_set_has_legacy_mapped_variants(db: Session, score_set: ScoreSet) -> bool: + """Whether the frozen ``MappedVariant`` table still holds current rows for this score set. + + Gates ``mapped/{urn}.mapped-variants.json``, the one artifact still sourced from that table. Score + sets mapped after the allele-substrate migration have no such rows. """ return ( db.scalars( @@ -148,9 +165,8 @@ def counts_csv(db: Session, score_set: ScoreSet) -> Optional[str]: def annotations_csv(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> str: """`csv/{urn}.annotations.csv` — every annotation namespace discovery offers for the score set. - The same *viewer* selects the namespaces and resolves their cells. Threading one viewer through both - is what keeps a calibration from being offered as a column group and then withheld as data, or the - reverse. + The same *viewer* selects the namespaces and resolves their cells, so a calibration can't be offered + as a column group and then withheld as data, or vice versa. """ return get_score_set_variants_as_csv( db, @@ -164,8 +180,11 @@ def annotations_csv(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationVi def mapped_variants_json(db: Session, score_set: ScoreSet) -> str: """`mapped/{urn}.mapped-variants.json` — the score set's current mapped variants. - Same shape as GET /api/v1/score-sets/{urn}/mapped-variants, but narrower: that endpoint also - returns superseded mappings, while this dump includes only each variant's current mapping. + Legacy, and the last artifact sourced from the frozen ``MappedVariant`` table. The endpoint this + once mirrored is gone — GET /score-sets/{urn}/mapped-variants now returns 410, and its replacement + serves a different field shape (see ``get_score_set_mapped_variants_removed``). + ``vrs/{urn}.vrs.ndjson`` supersedes this file in the archive. Only each variant's current mapping + is included, never a superseded one. """ mapped_variants = db.scalars( select(MappedVariant) @@ -180,33 +199,82 @@ def mapped_variants_json(db: Session, score_set: ScoreSet) -> str: def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: - """`va/{urn}.va.ndjson` — one record per current mapped variant at its highest materialized VA level. + """`va/{urn}.va.ndjson` — one record per annotatable variant at its highest materialized VA level. - Mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* streams. Every record is - newline-terminated, the last one included, so a line-based consumer needs no special case. + Mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* streams, which select the same set. + Every record is newline-terminated, including the last, so a line-based consumer needs no special + case. A variant the pipeline could not place carries no post-mapped allele, is therefore not + annotatable, and contributes no line; ``annotation`` is null where a placed variant yields no + VA-Spec layer. """ lines = [] - for mv in get_current_mapped_variants_for_annotation(db, score_set): - annotation = variant_highest_level_annotation(mv, principal=principal) + for variant, inputs in with_chunk_inputs( + get_annotatable_variants(db, score_set), lambda chunk: load_annotation_context_inputs(db, chunk) + ): + context = inputs.context_for(variant) + annotation = variant_highest_level_annotation(context, principal=principal) if context is not None else None record = { - "variant_urn": mv.variant.urn, - "annotation": annotation.model_dump(exclude_none=True) if annotation else None, + "variant_urn": variant.urn, + "annotation": serialize_annotation(annotation) if annotation else None, } lines.append(json.dumps(record, default=str)) return "".join(line + "\n" for line in lines) +def vrs_ndjson(db: Session, score_set: ScoreSet) -> str: + """`vrs/{urn}.vrs.ndjson` — GA4GH VRS objects for each mapped variant, one line each. + + Carries the VRS pair plus the Cat-VRS categorical variant, built from the same live allele links + `GET /variants/{urn}` serves. Its own artifact rather than CSV columns because these are nested + objects; join to `annotations.csv` on `mavedb.post_mapped_vrs_id`. + + `pre_mapped` is the assayed-level VRS on the target's own reference; `post_mapped` is the measured + allele lifted to a genomic or transcript reference. `categorical_variant` is spec-pure Cat-VRS (no + MaveDB fields), anchored on the measured allele with the derived alleles as members — null when + there's no hydratable authoritative allele. + + Emitted for every mapped variant regardless of calibration visibility, since none of this is + calibration-derived. + """ + lines = [] + + def load(chunk): + record_links = get_live_record_allele_links(db, [variant.id for variant in chunk]) + alleles = [link.allele for entry in record_links.values() for link in entry.links] + return record_links, get_allele_cross_references(db, alleles) + + for variant, (record_links, cross_references) in with_chunk_inputs(get_annotatable_variants(db, score_set), load): + entry = record_links.get(variant.id) + links = entry.links if entry is not None else [] + authoritative = next((link.allele for link in links if link.is_authoritative), None) + transit = build_categorical_variant(links, name=variant.urn or "", cross_references=cross_references) + + lines.append( + json.dumps( + { + "variant_urn": variant.urn, + "pre_mapped": entry.record.pre_mapped if entry is not None else None, + "post_mapped": authoritative.post_mapped if authoritative is not None else None, + "categorical_variant": ( + transit.categorical_variant.model_dump(mode="json", exclude_none=True) + if transit is not None + else None + ), + }, + default=str, + ) + ) + + return "".join(line + "\n" for line in lines) + + def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) -> Iterator[tuple[str, str]]: """Every archive entry one score set contributes, as ``(path within the zip, content)`` pairs. - Scores are unconditional. Counts appear only where count columns are defined, and the three - mapping-derived artifacts only where a current mapping exists — see the README's caveats, which - promise exactly this and are what a consumer checks a missing file against. - - A generator rather than a dict so the caller writes each artifact and lets it go. Returning them - together would hold all four of a score set's payloads in memory at once, and one score set's - ``va.ndjson`` alone runs to tens of kilobytes per variant once a pathogenicity layer materializes. + Scores are unconditional; counts and the mapping-derived artifacts appear only when the score set + has them — see the README's caveats. A generator rather than a dict so the caller writes and + releases each artifact, instead of holding a score set's full payload in memory at once. """ base = archive_path_base(str(score_set.urn)) viewer = principal.viewer_for(ScoreCalibrationViewer) @@ -215,35 +283,45 @@ def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) if score_set_has_current_mappings(db, score_set): yield f"csv/{base}.annotations.csv", annotations_csv(db, score_set, viewer) - yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) + yield f"vrs/{base}.vrs.ndjson", vrs_ndjson(db, score_set) yield f"va/{base}.va.ndjson", va_ndjson(db, score_set, principal) + if _emit_legacy_mapped_variants_artifact() and score_set_has_legacy_mapped_variants(db, score_set): + yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) + counts = counts_csv(db, score_set) if counts is not None: yield f"csv/{base}.counts.csv", counts def public_experiment_set( - experiment_set_view: ExperimentSetPublicDump, visible_calibration_ids: set[int] + experiment_set_view: ExperimentSetPublicDump, + visible_calibration_ids: set[int], + readable_related_urns: set[str], ) -> Optional[ExperimentSetPublicDump]: - """ - Narrow a validated experiment set to what belongs in the public dump. - - Drops calibrations an anonymous caller may not read, then experiments left with no score sets, and - returns None for an experiment set left with no experiments. The score sets themselves need no filter: - the loading query already restricts them to published, CC0-licensed ones. - - Narrowing the validated view rather than the ORM graph is deliberate. ``ExperimentSet.experiments`` and - ``ScoreSet.score_calibrations`` are both mapped with ``cascade="all, delete-orphan"``, so removing a - member from either ORM collection marks the removed row as an orphan and the next flush deletes it. - This script can flush: ``with_database_session`` commits when invoked with ``--commit``. - - Args: - experiment_set_view (ExperimentSetPublicDump): The validated experiment set to narrow. - visible_calibration_ids (set[int]): Ids of the calibrations an anonymous caller may read. - - Returns: - Optional[ExperimentSetPublicDump]: The narrowed experiment set, or None if nothing public remains. + """Narrow a validated experiment set to what belongs in the public dump. + + Drops calibrations the caller may not READ, blanks a superseding score set the caller may not READ, + drops meta-analyses the caller may not READ from `meta_analyzed_by_score_set_urns`, drops experiments + left with no score sets, and returns None if no experiments remain. The score sets themselves need no + filter — the loading query already restricts them to published, CC0-licensed rows. + + Blanking supersession matters because a published score set is often superseded by a *private* + in-progress replacement; without it, `main.json` would name an unreleased score set's URN and title + in an archive published to Zenodo, which can't be recalled. + + `readable_related_urns` holds the superseding and meta-analysis score sets the caller may READ, the + rule `_score_set_response` applies. It is defense-in-depth rather than the live gate, and it can only + subtract: `published_experiment_sets` loads members through a published+CC0 filter that propagates to + `superseding_score_set` and `meta_analyzed_by_score_sets`, so a private replacement or meta-analysis is + already gone before this runs — and so is a *published* successor whose license keeps it out of the + archive. That second case is unresolved: the consumer reads a superseded score set as current. Naming + it would need the successor fetched by a separate unfiltered query and injected into the view, not + merely left un-blanked; whether it should be named is an open policy question. Both behaviors are + pinned by `test_supersession_is_named_only_when_the_archive_carries_the_successor`. + + Narrows the validated view rather than the ORM graph because `ExperimentSet.experiments` and + `ScoreSet.score_calibrations` cascade-delete orphans, and this script can flush (`--commit`). """ experiments = [] for experiment_view in experiment_set_view.experiments: @@ -257,7 +335,16 @@ def public_experiment_set( calibration for calibration in (score_set_view.score_calibrations or []) if calibration.id in visible_calibration_ids - ] + ], + "superseding_score_set": ( + score_set_view.superseding_score_set + if score_set_view.superseding_score_set is not None + and score_set_view.superseding_score_set.urn in readable_related_urns + else None + ), + "meta_analyzed_by_score_set_urns": [ + urn for urn in score_set_view.meta_analyzed_by_score_set_urns if urn in readable_related_urns + ], } ) for score_set_view in experiment_view.score_sets @@ -271,11 +358,11 @@ def public_experiment_set( def published_experiment_sets(db: Session) -> list[ExperimentSet]: - """Every published experiment set, with its members narrowed to what the dump may carry. + """Every published experiment set, with members narrowed to what the dump may carry. - The narrowing is in the loader rather than applied afterwards, so an unpublished experiment or a - non-CC0 score set is never loaded onto the graph the metadata view is validated from. An experiment - set can survive this with no members left; ``public_experiment_set`` drops those. + Narrowing happens in the loader so an unpublished experiment or non-CC0 score set is never loaded + onto the graph the metadata view validates from. An experiment set can still end up with no members + left; `public_experiment_set` drops those. """ return list( db.scalars( @@ -300,13 +387,11 @@ def published_experiment_sets(db: Session) -> list[ExperimentSet]: def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[str]]: """The `main.json` payload, and the score-set URNs whose artifacts the archive carries. - One function for both because they are one decision: a score set is in the archive exactly when its - metadata survived narrowing, so deriving the URN list from the narrowed views rather than from the - query keeps the two from disagreeing. + One function for both: a score set is in the archive exactly when its metadata survives narrowing, + so deriving the URN list from the narrowed views (not the query) keeps the two in sync. - Publishing a score set does not publish its calibrations — a calibration keeps its own `private` flag - and a stricter READ rule — so which calibrations appear is asked of *principal* rather than inferred - from the score set's own visibility. + Calibration visibility is asked of *principal* rather than inferred from score-set visibility, since + publishing a score set doesn't publish its calibrations. """ experiment_sets = published_experiment_sets(db) @@ -329,10 +414,29 @@ def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[ # Issue: https://github.com/VariantEffect/mavedb-api/issues/192 # See, for instance, https://stackoverflow.com/questions/12670395/json-encoding-very-long-iterators. + # Asked of the ORM graph, not the validated views: a successor or meta-analysis is typically outside the + # loading query's published+CC0 filter, so checking the view would miss it. READ is the gate, but for + # successors this set only ever subtracts — the loader nulls a non-CC0 successor on the view first, so + # being readable here is not enough to get one named. See `public_experiment_set`. + loaded_score_sets = list(flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets)) + readable_related_urns = readable_score_set_urns( + principal.user_data, + [ + related + for score_set_orm in loaded_score_sets + for related in [ + *([score_set_orm.superseding_score_set] if score_set_orm.superseding_score_set is not None else []), + *score_set_orm.meta_analyzed_by_score_sets, # type: ignore[attr-defined] # backref + ] + ], + ) + experiment_set_views = [ narrowed for narrowed in ( - public_experiment_set(ExperimentSetPublicDump.model_validate(es), visible_calibration_ids) + public_experiment_set( + ExperimentSetPublicDump.model_validate(es), visible_calibration_ids, readable_related_urns + ) for es in experiment_sets ) if narrowed is not None @@ -356,25 +460,23 @@ def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[ def write_public_dump(db: Session, principal: Principal, archive: ZipFile) -> list[str]: """Write every member of the public dump into *archive*, and report the score sets carried. - Takes the archive rather than a filename so the whole composition — metadata, resources, and each - score set's artifacts — can be exercised without touching the filesystem. + Takes the archive rather than a filename so the whole composition can be exercised without touching + the filesystem. """ metadata, score_set_urns = public_dump_metadata(db, principal) - - # Metadata for all data sets goes in a single JSON file. archive.writestr("main.json", json.dumps(jsonable_encoder(metadata))) - # Copy the CC0 license and README. resources_dir = os.path.join(os.path.dirname(__file__), "resources") archive.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") archive.write(os.path.join(resources_dir, "README.md"), "README.md") + archive.write(os.path.join(resources_dir, "CHANGELOG.md"), "CHANGELOG.md") num_score_sets = len(score_set_urns) for i, score_set_urn in enumerate(score_set_urns): score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() if score_set is None: - # `main.json` already names this score set, so skipping it silently would leave the archive - # advertising files it does not contain. Reachable only if the row disappears mid-run. + # main.json already names this score set, so skip-silently would advertise files it doesn't + # contain. Reachable only if the row disappears mid-run. logger.warning( f"[{i + 1}/{num_score_sets}] {score_set_urn} is named in main.json but could no longer be " "loaded; the archive will carry no files for it." diff --git a/src/mavedb/scripts/export_sweep.py b/src/mavedb/scripts/export_sweep.py index 792469850..fb04b55a7 100644 --- a/src/mavedb/scripts/export_sweep.py +++ b/src/mavedb/scripts/export_sweep.py @@ -14,7 +14,7 @@ Writes one CSV row per attempted (score set, surface) pair, successes included: a report showing only failures cannot distinguish "nothing broke" from "nothing ran". Exits non-zero if any pair failed. -Every current mapped variant of every published score set is attempted. The measured cost of sweeping all +Every annotatable variant of every published score set is attempted. The measured cost of sweeping all score set level surfaces at the current database size is well under an hour for CSV surfaces and a little over an hour for VA annotation surfaces. @@ -43,13 +43,14 @@ variant_study_result, ) from mavedb.lib.annotation.conformance import AnnotationRoundTripError, round_trip_annotation +from mavedb.lib.alleles import with_chunk_inputs +from mavedb.lib.annotation.context import VariantAnnotationContext, load_annotation_context_inputs from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation +from mavedb.lib.score_sets import get_annotatable_variants from mavedb.lib.urns import variant_urn_sort_key -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.scripts.environment import script_environment, with_database_session @@ -78,7 +79,7 @@ ] -def _surfaces(principal: Principal) -> list[tuple[str, Callable[[MappedVariant], Optional[Any]]]]: +def _surfaces(principal: Principal) -> list[tuple[str, Callable[[VariantAnnotationContext], Optional[Any]]]]: """The three annotation surfaces the API streams, named as their endpoint path segments.""" return [ ("study-result", variant_study_result), @@ -89,7 +90,7 @@ def _surfaces(principal: Principal) -> list[tuple[str, Callable[[MappedVariant], @dataclass class SurfaceResult: - """What one annotation surface did across every current mapped variant of one score set.""" + """What one annotation surface did across every annotatable variant of one score set.""" score_set_urn: str surface: str @@ -101,7 +102,7 @@ class SurfaceResult: first_failing_variant_urn: str = "" exception_class: str = "" message: str = "" - #: Set when the pair was never attempted, e.g. the score set has no current mapped variants. + #: Set when the pair was never attempted, e.g. the score set has no annotatable variants. skip_reason: str = "" def record_failure(self, variant_urn: str, outcome: str, err: BaseException) -> None: @@ -145,12 +146,17 @@ class SweepTotals: def sweep_annotation_surface( + db: Session, score_set_urn: str, surface: str, - annotate: Callable[[MappedVariant], Optional[Any]], - mapped_variants: list[MappedVariant], + annotate: Callable[[VariantAnnotationContext], Optional[Any]], + variants: list[Variant], ) -> SurfaceResult: - """Attempt one annotation surface across every current mapped variant of one score set. + """Attempt one annotation surface across every annotatable variant of one score set. + + The contexts are loaded a chunk at a time and built per variant per surface, rather than once and reused, + because that is what the streaming endpoints do per request — a defect in context construction is a defect + in what the API serves, and the sweep only sees it by taking the same path. Nothing raises out of here. A sweep that aborted on the first bad variant would report the corpus as far healthier than it is. @@ -158,14 +164,15 @@ def sweep_annotation_surface( result = SurfaceResult( score_set_urn=score_set_urn, surface=surface, - variants_attempted=len(mapped_variants), + variants_attempted=len(variants), ) - for mapped_variant in mapped_variants: - variant_urn = getattr(mapped_variant.variant, "urn", "") or "" + for variant, inputs in with_chunk_inputs(variants, lambda chunk: load_annotation_context_inputs(db, chunk)): + variant_urn = getattr(variant, "urn", "") or "" try: - annotation = annotate(mapped_variant) + context = inputs.context_for(variant) + annotation = annotate(context) if context is not None else None except EXPECTED_ABSENCE_EXCEPTIONS: # An expected absence, drawn from the same definition the streaming endpoints use so the two # cannot disagree. Counting it as a failure would bury real defects under millions of @@ -347,7 +354,7 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st logger.info( f"Sweeping {len(score_sets)} published score sets across {len(surfaces) + 1} surfaces, " - "attempting every current mapped variant of each." + "attempting every annotatable variant of each." ) if max_score_sets is not None: logger.warning(f"Bounded to the first {max_score_sets} score sets by --max-score-sets; not full coverage.") @@ -356,21 +363,21 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st for index, score_set in enumerate(score_sets): urn = score_set.urn or f"" - mapped_variants = list(get_current_mapped_variants_for_annotation(db, score_set)) + variants = list(get_annotatable_variants(db, score_set)) variant_count = variant_counts.get(score_set.id, 0) surface_results: list[SurfaceResult] = [] # The annotation surfaces need a current mapping to have anything to say. The CSV surface does # not: it emits a row per variant and outer-joins the mapping, so an unmapped variant still gets # a row of NA columns. The two skip on different conditions for that reason. - if mapped_variants: + if variants: surface_results.extend( - sweep_annotation_surface(urn, surface, annotate, mapped_variants) for surface, annotate in surfaces + sweep_annotation_surface(db, urn, surface, annotate, variants) for surface, annotate in surfaces ) else: surface_results.extend( SurfaceResult( - score_set_urn=urn, surface=surface, outcome=SKIPPED, skip_reason="no current mapped variants" + score_set_urn=urn, surface=surface, outcome=SKIPPED, skip_reason="no annotatable variants" ) for surface, _ in surfaces ) @@ -382,9 +389,9 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st SurfaceResult(score_set_urn=urn, surface="score-set-csv", outcome=SKIPPED, skip_reason="no variants") ) - if mapped_variants or variant_count: + if variants or variant_count: totals.score_sets_attempted += 1 - totals.variants_attempted += len(mapped_variants) + totals.variants_attempted += len(variants) else: totals.score_sets_skipped += 1 @@ -418,7 +425,7 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st logger.info(f"Wrote {len(rows)} rows to {output_path}") logger.info( f"Score sets: {totals.score_sets_published} published, {totals.score_sets_attempted} attempted, " - f"{totals.score_sets_skipped} skipped with no current mapped variants." + f"{totals.score_sets_skipped} skipped with no annotatable variants." ) logger.info(f"Variants: {totals.variants_attempted} attempted per surface, every one held by those score sets.") logger.info("Outcomes: " + ", ".join(f"{outcome}={count}" for outcome, count in sorted(outcomes.items()))) diff --git a/src/mavedb/scripts/pipeline_tracking.py b/src/mavedb/scripts/pipeline_tracking.py new file mode 100644 index 000000000..15f7c4441 --- /dev/null +++ b/src/mavedb/scripts/pipeline_tracking.py @@ -0,0 +1,195 @@ +"""Operator-facing CLI for tracking which score sets need a pipeline run. + +Command +------- +list-score-sets + Produces a table of all score sets with their most recent pipeline run and + whether they need to be re-processed since a given deployment cutoff date. + +Usage: + + # Tracking list — show all published score sets and last pipeline run + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets + + # Filter to only those whose last run is before (or missing since) a deployment date + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets \\ + --needs-rerun-since 2026-05-01 + + # Include private score sets + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --include-private + + # JSON output for piping / spreadsheet import + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --json +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Optional + +import asyncclick as click +from sqlalchemy import Integer, cast, select +from sqlalchemy.orm import Session + +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + + +def _format_dt(dt: Optional[datetime]) -> str: + return dt.isoformat() if dt else "-" + + +def _needs_rerun(last_pipeline_finished: Optional[datetime], cutoff: Optional[datetime]) -> bool: + """Return True if the score set needs a re-run relative to *cutoff*.""" + if cutoff is None: + return False + if last_pipeline_finished is None: + return True + + # Normalise both to UTC-aware for comparison + aware_cutoff = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=timezone.utc) + aware_finished = ( + last_pipeline_finished if last_pipeline_finished.tzinfo else last_pipeline_finished.replace(tzinfo=timezone.utc) + ) + return aware_finished < aware_cutoff + + +def _last_pipeline_subquery(db: Session, score_set_id: int) -> Optional[Pipeline]: + """Return the most recently finished (or created) pipeline for *score_set_id*.""" + # score_set_id is stored as an integer in job_params JSONB + job_run_sq = ( + select(JobRun.pipeline_id) + .where(cast(JobRun.job_params["score_set_id"].astext, Integer) == score_set_id) + .where(JobRun.pipeline_id.is_not(None)) + .distinct() + .subquery() + ) + + return db.scalars( + select(Pipeline).where(Pipeline.id.in_(select(job_run_sq))).order_by(Pipeline.created_at.desc()).limit(1) + ).one_or_none() + + +def _build_score_set_rows( + db: Session, + score_sets: list[ScoreSet], + cutoff: Optional[datetime], +) -> list[dict]: + rows = [] + for ss in score_sets: + last_pipeline = _last_pipeline_subquery(db, ss.id) + last_pipeline_name = last_pipeline.name if last_pipeline else None + last_pipeline_status = str(last_pipeline.status) if last_pipeline else None + last_finished = last_pipeline.finished_at if last_pipeline else None + last_created = last_pipeline.created_at if last_pipeline else None + + rows.append( + { + "score_set_urn": ss.urn or "(no urn)", + "processing_state": str(ss.processing_state) if ss.processing_state else "-", + "mapping_state": str(ss.mapping_state) if ss.mapping_state else "-", + "num_variants": ss.num_variants, + "private": ss.private, + "last_pipeline_name": last_pipeline_name or "-", + "last_pipeline_status": last_pipeline_status or "-", + "last_pipeline_created_at": _format_dt(last_created), + "last_pipeline_finished_at": _format_dt(last_finished), + "needs_rerun": _needs_rerun(last_finished, cutoff), + } + ) + + return rows + + +@script_environment.command(name="list-score-sets") +@with_database_session +@click.option( + "--needs-rerun-since", + "needs_rerun_since", + default=None, + help=( + "ISO date/datetime of a deployment cutoff (e.g. 2026-05-01 or 2026-05-01T12:00:00). " + "Score sets whose last pipeline finished before this timestamp (or have never run) are " + "flagged as needing a re-run." + ), +) +@click.option("--include-private", is_flag=True, default=False, help="Include private (unpublished) score sets.") +@click.option("--needs-rerun-only", is_flag=True, default=False, help="Only show score sets that need a re-run.") +@click.option("--limit", type=int, default=None, help="Cap the number of rows returned (applied after all filtering).") +@click.option("--json", "as_json", is_flag=True, help="Emit results as JSON.") +def list_score_sets( + db: Session, + needs_rerun_since: Optional[str], + include_private: bool, + needs_rerun_only: bool, + limit: Optional[int], + as_json: bool, +) -> None: + """List all score sets with their last pipeline run and re-run status.""" + cutoff: Optional[datetime] = None + if needs_rerun_since: + try: + cutoff = datetime.fromisoformat(needs_rerun_since) + except ValueError: + click.echo( + f"Invalid --needs-rerun-since value: {needs_rerun_since!r}. Use ISO format e.g. 2026-05-01.", err=True + ) + raise SystemExit(1) + + query = select(ScoreSet).where(ScoreSet.urn.is_not(None)) + if not include_private: + query = query.where(ScoreSet.private == False) # noqa: E712 + query = query.order_by(ScoreSet.urn) + + score_sets = db.scalars(query).all() + rows = _build_score_set_rows(db, list(score_sets), cutoff) + + if needs_rerun_only: + rows = [r for r in rows if r["needs_rerun"]] + + if limit is not None: + rows = rows[:limit] + + if as_json: + click.echo(json.dumps(rows, indent=2, default=str)) + return + + if not rows: + click.echo("No score sets match the given filters.") + return + + needs_rerun_count = sum(1 for r in rows if r["needs_rerun"]) + click.echo(f"Total: {len(rows)} score set(s)" + (f", {needs_rerun_count} need re-run" if cutoff else "")) + if cutoff: + click.echo(f"Cutoff: {cutoff.isoformat()}") + click.echo() + + col_w = {"urn": 28, "ps": 28, "ms": 26, "nv": 8, "pipe": 28, "pipe_status": 12, "finished": 12} + header = ( + f"{'URN':<{col_w['urn']}} {'PROC_STATE':<{col_w['ps']}} {'MAP_STATE':<{col_w['ms']}} " + f"{'VARIANTS':>{col_w['nv']}} {'LAST_PIPELINE':<{col_w['pipe']}} " + f"{'PIPE_STATUS':<{col_w['pipe_status']}} {'LAST_FINISHED':<{col_w['finished']}}" + + (" RERUN?" if cutoff else "") + ) + click.echo(header) + click.echo("-" * len(header)) + + for r in rows: + rerun_flag = ("YES" if r["needs_rerun"] else "no") if cutoff else "" + click.echo( + f"{r['score_set_urn']:<{col_w['urn']}} " + f"{r['processing_state']:<{col_w['ps']}} " + f"{r['mapping_state']:<{col_w['ms']}} " + f"{r['num_variants']:>{col_w['nv']}} " + f"{r['last_pipeline_name']:<{col_w['pipe']}} " + f"{r['last_pipeline_status']:<{col_w['pipe_status']}} " + f"{r['last_pipeline_finished_at']:<{col_w['finished']}}" + (f" {rerun_flag}" if cutoff else "") + ) + + +if __name__ == "__main__": + script_environment() diff --git a/src/mavedb/scripts/project_measured_variants.py b/src/mavedb/scripts/project_measured_variants.py new file mode 100644 index 000000000..5e0b8ae19 --- /dev/null +++ b/src/mavedb/scripts/project_measured_variants.py @@ -0,0 +1,108 @@ +"""Write the measured projection group of every live mapping record, score set by score set. + +New mappings get their groups from the reverse translation job. This covers records that have not been +through it since projection groups existed, notably those the substrate backfill reshaped from legacy +mappings, which carry no groups. Only coordinates are needed (cdot and UTA), not the mapper, so it is fast +enough to run straight after a release. Re-running is safe: unchanged groups are left alone. + +Rewriting a changed group detaches the alleles reverse translation added to it, the measured variant's +other-level partner and its protein consequence, along with their ClinVar, gnomAD and VEP annotations. +Each score set's log line counts them as ``detached_rt_links``; re-run reverse translation on every score +set with a non-zero count. ``--missing-only`` only writes groups that do not exist yet, so it never +detaches anything. + +With ``--commit``, each score set commits as it finishes, so an interrupted run keeps its progress and a run +against a serving database holds row locks on one score set at a time. ``--shard`` splits the score sets +between concurrent processes. + +Usage: + python -m mavedb.scripts.project_measured_variants --dry-run --score-set urn:mavedb:00000663-a-1 + python -m mavedb.scripts.project_measured_variants --commit --shard 0/4 + python -m mavedb.scripts.project_measured_variants --commit --missing-only +""" + +import logging +import time +from collections import Counter +from typing import Optional + +import asyncclick as click +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.data_providers.services import cdot_rest +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import DatabaseSessionAction, script_environment, with_database_session +from mavedb.worker.jobs.variant_processing.reverse_translation import cdna_transcript_resolver +from mavedb.worker.lib.projection import project_measured_variants +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def _parse_shard(shard: Optional[str]) -> Optional[tuple[int, int]]: + if shard is None: + return None + index, _, count = shard.partition("/") + if not (index.isdigit() and count.isdigit()) or not 0 <= int(index) < int(count): + raise click.BadParameter("expected INDEX/COUNT, e.g. 0/4", param_hint="--shard") + return int(index), int(count) + + +@script_environment.command() +@with_database_session(pass_action=True) +@click.option("--score-set", "urns", multiple=True, help="Only this score set (URN). Repeatable.") +@click.option( + "--shard", default=None, help="Process only every COUNT-th score set by size, starting at INDEX, as INDEX/COUNT." +) +@click.option( + "--missing-only", + is_flag=True, + help="Only records with no measured projection group yet. Never detaches reverse translation alleles.", +) +def project_measured_variants_command( + db: Session, action: DatabaseSessionAction, urns: tuple[str, ...], shard: Optional[str], missing_only: bool +) -> None: + """Write measured projection groups for every score set with variants.""" + shard_spec = _parse_shard(shard) + # Largest first, dealt round-robin, so shards finish together rather than one holding the big score sets. + query = ( + select(ScoreSet.id, ScoreSet.urn) + .where(ScoreSet.num_variants > 0) + .order_by(ScoreSet.num_variants.desc(), ScoreSet.id) + ) + if urns: + query = query.where(ScoreSet.urn.in_(urns)) + score_sets = [ + (id_, urn) + for position, (id_, urn) in enumerate(db.execute(query).tuples()) + if shard_spec is None or position % shard_spec[1] == shard_spec[0] + ] + logger.info(f"Projecting measured variants for {len(score_sets)} score sets.") + + coordinates = WorkerCoordinateTranslator(cdot_rest()) + totals: Counter[str] = Counter() + started = time.monotonic() + for index, (score_set_id, urn) in enumerate(score_sets, start=1): + counts = project_measured_variants( + db, + score_set_id, + coordinates=coordinates, + resolve_transcript=cdna_transcript_resolver(db, score_set_id, coordinates), + missing_only=missing_only, + ) + if action is DatabaseSessionAction.COMMIT: + db.commit() + else: + db.flush() + db.expunge_all() + totals.update(counts) + logger.info(f"[{index}/{len(score_sets)}] {urn}: {dict(counts)}") + + elapsed = time.monotonic() - started + logger.info(f"Done in {elapsed:.0f}s: {dict(totals)}") + + +if __name__ == "__main__": + project_measured_variants_command() diff --git a/src/mavedb/scripts/resources/CHANGELOG.md b/src/mavedb/scripts/resources/CHANGELOG.md new file mode 100644 index 000000000..2bac1a66f --- /dev/null +++ b/src/mavedb/scripts/resources/CHANGELOG.md @@ -0,0 +1,117 @@ +# MaveDB Public Data Dump — Changelog + +This file records what changed between versions of the MaveDB bulk data archive +published on [Zenodo](https://doi.org/10.5281/zenodo.11201736). It ships inside +every archive so the version you downloaded always carries its own history. + +Versions are the Zenodo archive versions (v1, v2, …), not MaveDB API or website +release numbers. The `asOf` timestamp in `main.json` records the exact moment a +given archive was generated. + +--- + +## [Unreleased] + +### Added + +- **VRS objects** — `vrs/{urn}.vrs.ndjson` for every score set that has completed + the mapping pipeline. One record per variant placed on a reference, carrying the `pre_mapped` / + `post_mapped` GA4GH VRS pair and the Cat-VRS `categorical_variant`. Nested objects + cannot live in a CSV cell, so they get their own artifact; join to + `csv/{urn}.annotations.csv` on `mavedb.post_mapped_vrs_id`. No part of this file is + withheld on score-calibration visibility grounds, because none of it is derived + from a calibration. + +### Deprecated + +- **`mapped/{urn}.mapped-variants.json`** — superseded by `vrs/{urn}.vrs.ndjson`. It + is sourced from the pre-allele-graph mapping store, which no longer receives + writes, so it now appears only for score sets mapped before that migration. It is + kept for one deprecation window so existing consumers are not broken by a single + release, but will be removed in the next major release. The old single-mapping + shape encodes one representation per variant, a cardinality the allele graph does + not have (one variant maps to many alleles across coordinate levels), so + `vrs/{urn}.vrs.ndjson` is a re-shape, not a drop-in rename: migrate to it rather + than expecting the old file to keep the old shape. + +### Changed + +- **`va/{urn}.va.ndjson` follows VA-Spec 1.1.0 and Cat-VRS 1.1.1.** Field renames + consumers must follow: a study result's `focusVariant` is `focus`; a proposition's + `subjectVariant` is `subject`, and `objectCondition` / `objectSequenceFeature` are + `object`. The functional classification system is + `ga4gh-gkm-term:experimental-var-func-impact-classification`. +- The VA-Spec proposition subject is a Cat-VRS `CategoricalVariant` over the measured + change's other representations (its genomic or transcript twin and its protein + consequence) where it has any, with `mappings` to their ClinGen, gnomAD and ClinVar + records. +- `mavedb.post_mapped_hgvs_g` / `_c` / `_p` describe the measured variant and its + projections to the other levels, with the genomic string normalized. Score sets + assayed at the protein level fill only `mavedb.post_mapped_hgvs_p`, and their + `gnomad.*` and `clinvar.*` columns are `NA`: both are nucleotide-level resources. + A coding change across an exon junction has `NA` for `mavedb.post_mapped_hgvs_g`. +- gnomAD frequencies are now reported from whichever release each allele's current + record came from, rather than only from the single release the deployment serves. + An allele not covered by a newer release previously reported `NA` for every gnomAD + column; it now reports the frequency MaveDB actually holds, with + `gnomad.gnomad_version` naming its release. Expect a mixture of releases within one + file and read that column per row. + +### Fixed + +- The three mapping-derived artifacts are no longer omitted for score sets mapped + after the allele-graph migration. +- `main.json` no longer names an unpublished superseding score set's URN and title. + +--- + +## [5] — 2026-06-25 + +Version 5 is the first version tracked in this changelog. Versions 1–4 contained +metadata (`main.json`) plus per-score-set score and count CSVs; they predate this +file and are not itemized here. Version 5 adds variant-mapping and annotation +outputs derived from the MaveDB variant mapping pipeline. + +### Added + +- **Annotation CSVs** — `csv/{urn}.annotations.csv` for every score set that has + completed the mapping pipeline. Joins external-database annotations (Ensembl VEP + functional consequence, gnomAD v4.1 allele frequency, ClinGen Allele Registry ID) + with post-mapped HGVS (`g.`/`c.`/`p.` and assay-level) and the GA4GH VRS digest. +- **Mapped-variant JSON** — `mapped/{urn}.mapped-variants.json` for every mapped + score set. Each record carries the pre-mapped and post-mapped VRS alleles, the + VRS schema version, the mapping API version, and the ClinGen allele ID, mirroring + `GET /api/v1/score-sets/{urn}/mapped-variants`. +- **VA-Spec annotation NDJSON** — `va/{urn}.va.ndjson` for every mapped score set. + One line per variant currently placed on a reference, carrying its highest materialized GA4GH + VA-Spec layer (study result → functional-impact statement → pathogenicity + statement), mirroring the `annotated-variants` streaming endpoints. +- **ClinVar release columns** in the annotation CSVs, covering the `2015_02` + through `2026_01` annual snapshots. + +### Changed + +- **CSV columns are now namespaced.** Every column carries a source prefix + separated by a dot: score columns are `scores.*` (e.g. `scores.score`), counts + are `counts.*`, mapping-pipeline columns are `mavedb.*`, and external annotations + are `vep.*` / `gnomad.*` / `clingen.*`. Core identifier columns (`accession`, + `hgvs_nt`, `hgvs_pro`, `hgvs_splice`) remain unprefixed. **Consumers that parsed + earlier archives by bare column name (e.g. `score`) must update to the namespaced + names.** +- **Cis-phased (haplotype) variants** are now combined into a single HGVS + expression in the CSV outputs where a combined expression can be derived. +- Mapped-variant and VA-Spec outputs include only the **current** mapping for each + variant; superseded mappings are excluded. + +### Fixed + +- VA-Spec annotation output is now valid and round-trippable (it deserializes back + into GA4GH VA-Spec objects without loss). + +### Notes + +- Annotation, mapped-variant, and VA-Spec files are present **only** for score sets + that have completed the mapping pipeline. Unmapped score sets ship scores/counts + CSVs only, as before. +- See `README.md` in this archive for the full file layout, column definitions, and + caveats. diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index a3f5148f0..e28e5541a 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -30,6 +30,7 @@ Unpublished data, private datasets, and datasets published under other licenses ``` mavedb-dump.YYYYMMDDHHMMSS.zip ├── README.md # This file +├── CHANGELOG.md # What changed between archive versions ├── LICENSE.txt # Creative Commons CC0 1.0 license text ├── main.json # Metadata for all included datasets ├── csv/ @@ -37,11 +38,14 @@ mavedb-dump.YYYYMMDDHHMMSS.zip │ ├── {urn}.counts.csv # Variant counts (score sets with count data only) │ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, ClinGen and ClinVar, plus score calibration interpretations │ # (score sets that have completed mapping only) -├── mapped/ -│ └── {urn}.mapped-variants.json # Mapped variant data including VRS alleles and HGVS +├── vrs/ +│ └── {urn}.vrs.ndjson # GA4GH VRS and Cat-VRS objects, one record per variant placed on a reference │ # (score sets that have completed mapping only) +├── mapped/ +│ └── {urn}.mapped-variants.json # Legacy; superseded by vrs/. Score sets mapped before the +│ # allele-graph migration only └── va/ - └── {urn}.va.ndjson # GA4GH VA-Spec annotations, one record per mapped variant + └── {urn}.va.ndjson # GA4GH VA-Spec annotations, one record per variant placed on a reference # (score sets that have completed mapping only) ``` @@ -136,9 +140,18 @@ Columns are grouped by a namespace prefix. The groups below always appear: | `gnomad.gnomad_faf95_max` | Maximum filtering allele frequency at 95% confidence across genetic ancestry groups | | `gnomad.gnomad_faf95_max_ancestry` | Genetic ancestry group attaining `gnomad_faf95_max` | | `gnomad.gnomad_id` | gnomAD variant identifier (`chrom-pos-ref-alt`), e.g. `17-43092919-G-A` | -| `gnomad.gnomad_version` | gnomAD release the frequencies were drawn from (e.g. `v4.1`) | +| `gnomad.gnomad_version` | gnomAD release *this row's* frequencies were drawn from (e.g. `v4.1`). Varies by row — see Caveats | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | +The `mavedb.post_mapped_hgvs_*` columns describe the measured variant at each level: the measured HGVS +plus its projections to the other levels. The genomic string is normalized, so for a genomic assay it can +differ from `mavedb.post_mapped_hgvs_at_assay_level`, and a projected inversion is written as a `delins`. +A coding change across an exon junction (a codon split by an intron) has no single genomic change, so its +`mavedb.post_mapped_hgvs_g` is `NA`. +Score sets assayed at the protein level fill only `mavedb.post_mapped_hgvs_p`: a protein change has +no single coding or genomic encoding. gnomAD and ClinVar are nucleotide-level resources, so the `gnomad.*` +and `clinvar.*` columns are `NA` for protein-level variants. + Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. Read the header rather than assuming a fixed column set. @@ -178,15 +191,56 @@ strength that the GA4GH VA-Spec has no equivalent for. The same variant's record therefore reports `moderate` where this file reports `MODERATE_PLUS`. Variants that could not be mapped, or for which a specific annotation is unavailable, will have -`NA` in the corresponding column. For multi-allelic variants (haplotypes), `mavedb.*` HGVS columns -will be `NA` because a single combined HGVS string cannot currently be derived. This may be updated in -a future release. +`NA` in the corresponding column. For a cis-phased multi-variant (haplotype), the measured-level HGVS is +a single combined expression (e.g. `NC_…:g.[123A>G;125T>C]`) when every member has an HGVS expression +on the same sequence, and `NA` otherwise. Haplotypes mapped with VRS 1.x have no VRS allele: they are absent from +`vrs/{urn}.vrs.ndjson` and `va/{urn}.va.ndjson`, `mavedb.post_mapped_vrs_id` and the projected levels +are `NA`, and the measured level carries only the HGVS their original mapping recorded, if any. + +### `vrs/{urn}.vrs.ndjson` + +[Newline-delimited JSON](https://ndjson.org/): one line per variant the pipeline placed onto a reference +sequence, carrying the GA4GH molecular objects for that variant and nothing else. Each line is: + +| Field | Description | +|-------|-------------| +| `variant_urn` | URN of the source variant — use this to join with `accession` in the CSV files | +| `pre_mapped` | VRS object using coordinates on the assay's own reference sequence (transcript or protein accession) | +| `post_mapped` | VRS object for the measured allele, lifted to a genomic or transcript reference | +| `categorical_variant` | GA4GH Cat-VRS `CategoricalVariant`, anchored on the measured allele with the derived alleles as members | + +A separate artifact rather than columns in `csv/{urn}.annotations.csv` because these are nested objects: a +CSV cell can carry an identifier or an HGVS string, but not a VRS object. Join the two on +`mavedb.post_mapped_vrs_id`, which is the same identifier as `post_mapped.id`. + +`categorical_variant` is spec-pure — it contains no MaveDB-specific fields. Its `mappings` cross-reference the +members' ClinGen allele IDs, gnomAD variant IDs and ClinVar variation IDs: `exactMatch` for records of the +measured change itself (in any coordinate frame), `relatedMatch` for records of other members, such as the +protein consequence or another nucleotide change encoding the same protein change. A gnomAD mapping names the +gnomAD release it was matched in as `coding.systemVersion`. Mappings are sorted by `coding.system`, then +`coding.code`. They carry identifiers only; frequencies and classifications are in +`csv/{urn}.annotations.csv`. A variant the pipeline could +not place carries no reference coordinates and is omitted from this file, so every line carries a non-null +`post_mapped` and the line count equals the number of variants with reference coordinates. `pre_mapped` is +`null` where no assay-frame VRS was recorded, and `categorical_variant` is `null` where the stored VRS +could not be resolved to a GA4GH object. + +Unlike `va/{urn}.va.ndjson`, nothing here is derived from a score calibration, so no part of this file is +withheld on calibration-visibility grounds. + +**Only present for score sets that have completed the MaveDB mapping pipeline.** + +--- ### `mapped/{urn}.mapped-variants.json` -A JSON array of the score set's **current** mapped variant records. The same shape as -`GET /api/v1/score-sets/{urn}/mapped-variants`, narrowed to `current: true`. Each record -corresponds to a single variant: +> **Legacy artifact, superseded by `vrs/{urn}.vrs.ndjson`.** This file is sourced from the +> pre-allele-graph mapping store, which no longer receives writes, so it is present only for score sets +> mapped before that migration and will be removed in a future release. `vrs/{urn}.vrs.ndjson` carries the +> same `preMapped`/`postMapped` pair for *every* mapped score set, plus the Cat-VRS object. New consumers +> should read that instead. + +A JSON array of the score set's **current** mapped variant records, one per variant: | Field | Description | |-------|-------------| @@ -204,14 +258,15 @@ corresponds to a single variant: `preMapped` and `postMapped` are raw GA4GH VRS objects (JSON). The `type` field within them may be `"Allele"`, `"Haplotype"`, or `"CisPhasedBlock"` depending on the variant. Records where mapping failed will have `preMapped: null`, `postMapped: null`, and a non-null `errorMessage`. **Only -present for score sets that have completed the MaveDB mapping pipeline.** +present for score sets mapped before the allele-graph migration** — see the note above. --- ### `va/{urn}.va.ndjson` -[Newline-delimited JSON](https://ndjson.org/): one line per current mapped variant. Each line is an -envelope mirroring the `GET /api/v1/score-sets/{urn}/annotated-variants/*` streaming endpoints: +[Newline-delimited JSON](https://ndjson.org/): one line per variant the pipeline currently places onto a +reference sequence. Each line is an envelope mirroring the +`GET /api/v1/score-sets/{urn}/annotated-variants/*` streaming endpoints: ```json {"variant_urn": "urn:mavedb:00000001-a-1#1", "annotation": { ... }} @@ -244,10 +299,17 @@ be combined with other evidence downstream, not as a standalone clinical verdict Research-use-only calibrations are excluded from this file, unlike `csv/{urn}.annotations.csv`, which includes them under a `research_use_only` flag. -`annotation` is `null` for current mapped variants that have no post-mapped allele (and therefore -cannot be annotated); the `variant_urn` is still present on those lines. Every current mapped variant -produces exactly one line, so the line count equals the current mapped-variant count. **Only present -for score sets that have completed the MaveDB mapping pipeline.** +The proposition subject is the measured change. When it has another representation (its genomic or +transcript twin, or its protein consequence) the subject is a Cat-VRS `CategoricalVariant` over those +representations, carrying the same `mappings` as `categorical_variant` in `vrs/{urn}.vrs.ndjson`. Unlike +that file, it leaves out other nucleotide changes that encode the same protein change, so it carries none of +their mappings. Otherwise the subject is the bare VRS allele and has no `mappings`. + +`annotation` is `null` where no VA-Spec layer could be built for a variant that does have reference +coordinates. A variant the pipeline could not place has none, and is omitted from this file entirely. +Every variant it currently places produces exactly one line, so the line count equals the number of +variants with current reference coordinates. **Only present for score sets that have completed the MaveDB +mapping pipeline.** --- @@ -258,9 +320,10 @@ for score sets that have completed the MaveDB mapping pipeline.** All files for a given score set share the same variant identifier: - In CSV files: the `accession` column (e.g. `urn:mavedb:00000001-a-1#42`) -- In `mapped-variants.json`: the `variantUrn` field +- In `vrs.ndjson` and `va.ndjson`: the `variant_urn` field +- In the legacy `mapped-variants.json`: the `variantUrn` field -To combine scores with annotations or with VRS data, join on `accession` = `variantUrn`. +To combine scores with annotations or with VRS data, join on `accession` = `variant_urn`. ### Linking files back to metadata @@ -295,8 +358,8 @@ score_set = next( - Only **published**, **CC0-licensed** data is included. Datasets with other licenses are not present in this dump even if they are publicly visible on MaveDB. -- Annotation files (`.annotations.csv`), mapped variant files (`.mapped-variants.json`), and - VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the +- Annotation files (`.annotations.csv`), VRS files (`.vrs.ndjson`), the legacy + `.mapped-variants.json` files, and VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the MaveDB variant mapping pipeline**. Score sets that have not yet been mapped, or for which mapping failed entirely, will not have these files. - `annotations.csv` includes **research-use-only** calibrations, flagged by `research_use_only`; the @@ -307,16 +370,20 @@ score_set = next( functional evidence only, not a full clinical ACMG determination. - Mapping is applied per variant within a score set. A score set that has completed the mapping pipeline may still contain individual variants with failed mappings. Those variants have `NA` in - all `mavedb.*`, `vep.*`, `gnomad.*`, and `clingen.*` columns in the annotations CSV, and - `preMapped: null` / `postMapped: null` in the JSON. + all `mavedb.*`, `vep.*`, `gnomad.*`, and `clingen.*` columns in the annotations CSV, are omitted + from the `vrs/` and `va/` files, and have `preMapped: null` / `postMapped: null` in the legacy JSON. - This dump is a snapshot of MaveDB's **current** state, not a historical archive. The `mapped/` JSON files include only each variant's current mapping — superseded records from earlier mapping - runs are not retained here. (Unlike this per-run dump, `GET /api/v1/score-sets/{urn}/mapped-variants` - does return superseded mappings, for callers that need that history directly.) The same applies to + runs are not retained here. (Callers that need mapping history directly can ask the API for a past + instant with the `as_of` parameter.) The same applies to `annotations.csv` and the `va/` files: annotations are always reported with respect to the current mapping object. If you need MaveDB's state as of a specific point in time, use a dump from that time (e.g. an earlier Zenodo-archived release) rather than looking for historical rows within one. -- gnomAD allele frequencies in `annotations.csv` are sourced from **gnomAD v4.1** specifically. +- gnomAD allele frequencies are **not all from one release**. MaveDB records a frequency per allele from + whichever gnomAD release last covered it, and a release that does not include an allele leaves that + allele's earlier frequency in place. Read `gnomad.gnomad_version` per row rather than assuming a single + release for the file; a mixture is normal and expected. (ClinVar differs: each release MaveDB holds gets + its own `clinvar.YYYY_MM` column group, so those coexist as columns rather than varying by row.) - `preMapped` VRS objects reference the assay's input sequence (a transcript or protein accession). `postMapped` VRS objects are remapped to the **GRCh38** reference genome. Do not compare coordinates between `preMapped` and `postMapped` directly. diff --git a/src/mavedb/scripts/run_job.py b/src/mavedb/scripts/run_job.py index 3204b14b4..55164235a 100644 --- a/src/mavedb/scripts/run_job.py +++ b/src/mavedb/scripts/run_job.py @@ -35,7 +35,7 @@ from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.worker.jobs.registry import STANDALONE_JOB_DEFINITIONS -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name from mavedb.worker.settings import RedisWorkerSettings from mavedb.worker.settings.lifecycle import standalone_ctx @@ -221,7 +221,9 @@ async def _enqueue_jobs( db.flush() arq_id = arq_job_id(job_run) - job = await redis.enqueue_job(job_run.job_function, job_run.id, _job_id=arq_id) + job = await redis.enqueue_job( + job_run.job_function, job_run.id, _job_id=arq_id, _queue_name=arq_queue_name(job_run) + ) if job: click.echo(f"Enqueued {job_name} (job_run={job_run.id}, arq_id={arq_id})") else: diff --git a/src/mavedb/scripts/run_pipeline.py b/src/mavedb/scripts/run_pipeline.py index c947fe122..658037fbb 100644 --- a/src/mavedb/scripts/run_pipeline.py +++ b/src/mavedb/scripts/run_pipeline.py @@ -11,7 +11,6 @@ poetry run python -m mavedb.scripts.run_pipeline --list """ -import datetime import logging import sys @@ -21,10 +20,9 @@ from mavedb.db.session import SessionLocal from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.score_set import ScoreSet from mavedb.models.user import User -from mavedb.worker.lib.managers.utils import arq_job_id from mavedb.worker.settings import RedisWorkerSettings logger = logging.getLogger(__name__) @@ -108,40 +106,25 @@ async def main( click.echo(f"User not found: {resolved_updater_id}", err=True) sys.exit(1) - correlation_id = f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" - pipeline_params: dict = { - "correlation_id": correlation_id, - "score_set_id": score_set.id, - "updater_id": user.id, - } - for key, value in extra_params: - pipeline_params[key] = value - - try: - pipeline_factory = PipelineFactory(session=db) - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=user, - pipeline_params=pipeline_params, - ) - except (KeyError, ValueError) as e: - click.echo(f"Failed to create pipeline: {e}", err=True) - sys.exit(1) - - click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={correlation_id})") - # Connect to Redis and enqueue redis = await create_pool(RedisWorkerSettings) try: - job = await redis.enqueue_job( - pipeline_entrypoint.job_function, - pipeline_entrypoint.id, - _job_id=arq_job_id(pipeline_entrypoint), + pipeline, job = await enqueue_pipeline_for_score_set( + db, + redis, + pipeline_name=pipeline_name, + score_set=score_set, + user=user, + extra_params={key: value for key, value in extra_params}, ) + click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={pipeline.correlation_id})") if job: click.echo(f"Enqueued start_pipeline job: {job.job_id}. Pipeline will execute asynchronously.") else: click.echo("Job was already enqueued (duplicate).", err=True) + except (KeyError, ValueError) as e: + click.echo(f"Failed to create pipeline: {e}", err=True) + sys.exit(1) finally: await redis.aclose() db.close() diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index bfe896eea..f27f38489 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -15,6 +15,12 @@ it (by hand, cron, or /loop) to keep driving progress; the heavy pipeline work happens entirely in the worker, with the Pipeline/JobRun tables as durable state. +Cohort order (gene cluster, then URN) is fixed across invocations, so a score set whose +latest run failed keeps its same spot on every re-run. Slots therefore always go to +untried score sets first; a failed one is only retried with whatever slots are left +over, so a handful of chronic failures can't camp on the concurrency window and starve +score sets that have never been attempted (see plan_enqueue). + Usage: # Preview what would be enqueued, without enqueuing anything. poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ @@ -28,6 +34,11 @@ poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ --phase caid --collection-urn urn:mavedb:collection-0000001 + # Re-register every allele with CAR while mapping and annotating; a CAID CAR now assigns + # differently replaces the stored one, and its gnomAD/ClinVar links are rebuilt in the same pipeline. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --published-only --concurrency 4 --force-reregister + # Run one gene you already know, no cohort files involved. poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ --gene BRCA1 --concurrency 4 @@ -64,7 +75,7 @@ from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.scripts.run_pipeline import _print_available_pipelines -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name from mavedb.worker.settings import RedisWorkerSettings logger = logging.getLogger(__name__) @@ -84,8 +95,6 @@ { "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } @@ -253,6 +262,7 @@ def plan_enqueue( *, in_flight_score_set_ids: set[int], current_score_set_ids: set[int], + failed_score_set_ids: frozenset[int] = frozenset(), slots: int, limit: Optional[int], ) -> list[tuple[ScoreSet, str, EnqueueDecision]]: @@ -261,29 +271,46 @@ def plan_enqueue( Slots are spent in cohort order, which cluster_cohort has already grouped by gene: a window therefore stays on one gene until that gene is exhausted, rather than splitting its concurrency across two genes where half the running pipelines warm a - ClinGen cache the other half never reads. + ClinGen cache the other half never reads. That ordering is fixed across invocations, + so a score set whose latest run failed sits at the same spot on every re-run. Filling + slots in a single pass over that order would let a handful of chronically-failing + score sets early in cluster/URN order camp on the window forever, since a FAILED + entry looks exactly like a never-run one to a single scan and keeps winning the same + slots run after run — starving entries later in the order that have never been + attempted. Untried entries are therefore filled first, in cohort order; only leftover + slots go to failed entries, also in cohort order (preserving cache-coherent + within-cluster fill for the retry pass too). Skip-current/skip-in-flight decisions never consume a slot; only "enqueue" does. """ - plan: list[tuple[ScoreSet, str, EnqueueDecision]] = [] + decisions: dict[Optional[int], EnqueueDecision] = {} remaining_slots = slots enqueued_count = 0 + def fill(entries: list[CohortEntry]) -> None: + nonlocal remaining_slots, enqueued_count + for entry in entries: + if remaining_slots <= 0 or (limit is not None and enqueued_count >= limit): + decisions[entry.score_set.id] = "skip_cap" + else: + decisions[entry.score_set.id] = "enqueue" + remaining_slots -= 1 + enqueued_count += 1 + + eligible: list[CohortEntry] = [] for entry in ordered_cohort: - score_set = entry.score_set - - if score_set.id in current_score_set_ids: - plan.append((score_set, entry.cluster_key, "skip_current")) - elif score_set.id in in_flight_score_set_ids: - plan.append((score_set, entry.cluster_key, "skip_in_flight")) - elif remaining_slots <= 0 or (limit is not None and enqueued_count >= limit): - plan.append((score_set, entry.cluster_key, "skip_cap")) + score_set_id = entry.score_set.id + if score_set_id in current_score_set_ids: + decisions[score_set_id] = "skip_current" + elif score_set_id in in_flight_score_set_ids: + decisions[score_set_id] = "skip_in_flight" else: - plan.append((score_set, entry.cluster_key, "enqueue")) - remaining_slots -= 1 - enqueued_count += 1 + eligible.append(entry) - return plan + fill([entry for entry in eligible if entry.score_set.id not in failed_score_set_ids]) + fill([entry for entry in eligible if entry.score_set.id in failed_score_set_ids]) + + return [(entry.score_set, entry.cluster_key, decisions[entry.score_set.id]) for entry in ordered_cohort] # --------------------------------------------------------------------------- @@ -385,6 +412,17 @@ def pipelines_by_score_set( return result +def latest_pipeline_by_score_set(db: Session, *, tracked_name: str, score_set_ids: list[int]) -> dict[int, Pipeline]: + """The most recently created tracked_name pipeline for each score set that has one. + + Shared by enqueue planning (was this score set's last attempt a failure?) and + reporting (what's its current status?), so a large cohort pays for this join once + per invocation instead of once per use. + """ + by_score_set = pipelines_by_score_set(db, tracked_name=tracked_name, score_set_ids=score_set_ids) + return {ss_id: max(pipelines, key=lambda p: p.created_at) for ss_id, pipelines in by_score_set.items()} + + def representative_error(db: Session, pipeline_id: int) -> Optional[str]: """Latest JobRun.error_message for a FAILED job under this pipeline.""" job_run = db.scalars( @@ -426,7 +464,7 @@ async def enqueue_pipeline( custom_pipeline: Optional[tuple[str, PipelineDefinition]], score_set: ScoreSet, user: User, - extra_params: tuple[tuple[str, str], ...], + extra_params: tuple[tuple[str, object], ...], ) -> EnqueueOutcome: """run_pipeline.py's enqueue body, generalized to accept either a base pipeline name or a resolved (name, job-subset) pair. A redis.enqueue_job failure triggers @@ -469,6 +507,7 @@ async def enqueue_pipeline( pipeline_entrypoint.job_function, pipeline_entrypoint.id, _job_id=arq_job_id(pipeline_entrypoint), + _queue_name=arq_queue_name(pipeline_entrypoint), ) except Exception as e: pipeline_factory.discard_pipeline(pipeline) @@ -565,6 +604,7 @@ def render_report( *, tracked_name: str, ordered_cohort: Sequence[CohortEntry], + latest_by_score_set: dict[int, Pipeline], in_flight_rows: list[tuple[Pipeline, Optional[int]]], current_since: Optional[datetime.date], ) -> tuple[str, list[str]]: @@ -575,20 +615,16 @@ def render_report( lines: list[str] = [] failed_urns: list[str] = [] - score_set_ids: list[int] = [entry.score_set.id for entry in ordered_cohort] # type: ignore[misc] - latest_by_score_set = pipelines_by_score_set(db, tracked_name=tracked_name, score_set_ids=score_set_ids) - lines.append(f"Cohort report for '{tracked_name}' ({len(ordered_cohort)} score sets):") lines.append(f"{'URN':<40} {'CLUSTER':<20} {'STATUS':<12} ERROR") for entry in ordered_cohort: score_set = entry.score_set cluster = entry.cluster_key or "-" - pipelines = latest_by_score_set.get(entry.score_set.id, []) # type: ignore[arg-type] - if not pipelines: + latest = latest_by_score_set.get(entry.score_set.id) # type: ignore[arg-type] + if latest is None: lines.append(f"{score_set.urn:<40} {cluster:<20} {'no run':<12}") continue - latest = max(pipelines, key=lambda p: p.created_at) error = "" if is_failure(latest.status): failed_urns.append(score_set.urn) # type: ignore[arg-type] @@ -679,6 +715,11 @@ def render_report( help="Write one URN per line for every cohort score set whose latest pipeline is FAILED/PARTIAL.", ) @click.option("--updater-id", type=int, default=None, help="ID of the user to attribute pipeline actions to.") +@click.option( + "--force-reregister", + is_flag=True, + help="Resubmit alleles that already hold a CAID to CAR, replacing any CAID CAR now assigns differently.", +) @click.option( "--extra-param", "extra_params", @@ -703,6 +744,7 @@ async def main( cohort_out: Optional[str], failure_out: Optional[str], updater_id: Optional[int], + force_reregister: bool, extra_params: tuple[tuple[str, str], ...], ) -> None: """Bulk-drive PIPELINE_NAME across a cohort of score sets. Use --list to see available pipelines.""" @@ -798,19 +840,27 @@ async def main( run_pipeline_name = None current_since_date = current_since.date() if current_since else None + cohort_score_set_ids: list[int] = [entry.score_set.id for entry in ordered_cohort] # type: ignore[misc] current_score_set_ids: set[int] = set() if current_since_date is not None: succeeded = pipelines_by_score_set( db, tracked_name=effective_name, - score_set_ids=[entry.score_set.id for entry in ordered_cohort], # type: ignore[misc] + score_set_ids=cohort_score_set_ids, statuses=[PipelineStatus.SUCCEEDED], ) for ss_id, pipelines in succeeded.items(): if any(is_current(p.status, p.finished_at, current_since_date) for p in pipelines): current_score_set_ids.add(ss_id) + latest_by_score_set = latest_pipeline_by_score_set( + db, tracked_name=effective_name, score_set_ids=cohort_score_set_ids + ) + failed_score_set_ids = frozenset( + ss_id for ss_id, pipeline in latest_by_score_set.items() if is_failure(pipeline.status) + ) + in_flight_rows = in_flight_pipelines(db, tracked_name=effective_name) in_flight_score_set_ids = {ss_id for _p, ss_id in in_flight_rows if ss_id is not None} @@ -819,6 +869,7 @@ async def main( ordered_cohort, in_flight_score_set_ids=in_flight_score_set_ids, current_score_set_ids=current_score_set_ids, + failed_score_set_ids=failed_score_set_ids, slots=slots, limit=limit, ) @@ -852,7 +903,7 @@ async def main( custom_pipeline=custom_pipeline, score_set=score_set, user=user, - extra_params=extra_params, + extra_params=extra_params + ((("force_reregister", True),) if force_reregister else ()), ) prefix = " [enqueued]" if outcome.ok else " [failed] " @@ -864,6 +915,7 @@ async def main( db, tracked_name=effective_name, ordered_cohort=ordered_cohort, + latest_by_score_set=latest_by_score_set, in_flight_rows=in_flight_rows, current_since=current_since_date, ) diff --git a/src/mavedb/scripts/variant_annotations.py b/src/mavedb/scripts/variant_annotations.py index ac9ff916c..fe19172c0 100644 --- a/src/mavedb/scripts/variant_annotations.py +++ b/src/mavedb/scripts/variant_annotations.py @@ -1,14 +1,17 @@ -"""Operator-facing CLI for inspecting variant annotation state. +"""Operator-facing CLI for inspecting a score set's current annotation status. + +The score set is the entry point (pipelines run per score set), but annotation status is a per-allele +fact: this resolves the score set's *current* alleles (and variants) through the live mapping links and +reports each subject's true current status — including statuses produced by another score set's run +that landed on a shared allele. It does **not** filter by which run wrote the event. Usage: - # Summarize annotation status counts for all variants in a score set poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 --json -For per-variant detail, query the v_variant_annotations view directly: - SELECT * FROM v_variant_annotations WHERE score_set_urn = 'urn:mavedb:00000001-a-1'; - SELECT * FROM v_variant_annotations WHERE variant_urn = 'urn:mavedb:00000001-a-1#42'; - SELECT * FROM v_variant_annotations WHERE annotation_status = 'failed'; +For per-subject detail, query the v_current_annotation_events view directly: + SELECT * FROM v_current_annotation_events WHERE allele_id = ; + SELECT * FROM v_current_annotation_events WHERE disposition = 'failed'; """ import json @@ -16,12 +19,13 @@ from typing import Optional import asyncclick as click -from sqlalchemy import func, select +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session +from mavedb.lib.clingen.alleles import get_alleles_for_score_set +from mavedb.models.annotation_event_view import CurrentAnnotationEventView from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.scripts.environment import script_environment, with_database_session logger = logging.getLogger(__name__) @@ -31,34 +35,52 @@ def _get_score_set(db: Session, urn: str) -> Optional[ScoreSet]: return db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() +def current_annotation_summary(db: Session, score_set: ScoreSet) -> list[dict]: + """Current annotation disposition counts for a score set, by annotation type. + + Resolves the score set's subjects — its current alleles (via the live mapping links) for + allele-subject types, and its variants for variant-subject types (mapping/RT/LDH) — then counts the + current status of each in ``v_current_annotation_events``. Allele-subject counts are per allele + (shared alleles counted once), reflecting the true current status regardless of which run produced + it; variant-subject counts are per variant. + """ + allele_ids = {row.allele_id for row in get_alleles_for_score_set(db, score_set.id)} + variant_ids = set(db.scalars(select(Variant.id).where(Variant.score_set_id == score_set.id)).all()) + + if not allele_ids and not variant_ids: + return [] + + rows = db.execute( + select( + CurrentAnnotationEventView.annotation_type, + CurrentAnnotationEventView.disposition, + func.count().label("count"), + ) + .where( + or_( + CurrentAnnotationEventView.allele_id.in_(allele_ids), + CurrentAnnotationEventView.variant_id.in_(variant_ids), + ) + ) + .group_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + .order_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + ).all() + + return [{"annotation_type": r.annotation_type, "disposition": r.disposition, "count": r.count} for r in rows] + + @script_environment.command(name="show-score-set") @with_database_session @click.argument("score_set_urn") @click.option("--json", "as_json", is_flag=True, help="Emit result as JSON.") def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: - """Summarize annotation status counts for all variants in a score set.""" + """Summarize the current annotation status of a score set's alleles and variants.""" score_set = _get_score_set(db, score_set_urn) if score_set is None: click.echo(f"Score set not found: {score_set_urn}", err=True) raise SystemExit(1) - # Count current annotation statuses grouped by annotation_type and status - rows = db.execute( - select( - VariantAnnotationStatus.annotation_type, - VariantAnnotationStatus.status, - func.count().label("count"), - ) - .join(Variant, Variant.id == VariantAnnotationStatus.variant_id) - .where( - Variant.score_set_id == score_set.id, - VariantAnnotationStatus.current == True, # noqa: E712 - ) - .group_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - ).all() - - # Total variant count for the score set + summary = current_annotation_summary(db, score_set) total_variants = db.scalar(select(func.count()).where(Variant.score_set_id == score_set.id)) or 0 if as_json: @@ -67,9 +89,7 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: { "score_set_urn": score_set_urn, "total_variants": total_variants, - "annotation_summary": [ - {"annotation_type": r.annotation_type, "status": r.status, "count": r.count} for r in rows - ], + "annotation_summary": summary, }, indent=2, ) @@ -77,12 +97,12 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: return click.echo(f"Score set: {score_set_urn} ({total_variants} variants)") - click.echo(f"\n{'ANNOTATION TYPE':<32} {'STATUS':<10} COUNT") - for r in rows: - click.echo(f"{r.annotation_type:<32} {str(r.status):<10} {r.count}") + click.echo(f"\n{'ANNOTATION TYPE':<32} {'DISPOSITION':<16} COUNT") + for r in summary: + click.echo(f"{r['annotation_type']:<32} {str(r['disposition']):<16} {r['count']}") - if not rows: - click.echo("No annotation status records found.") + if not summary: + click.echo("No current annotation events found for this score set.") if __name__ == "__main__": diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 880bfcfe1..1270a47ac 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -3,7 +3,7 @@ import uvicorn from eutils._internal.exceptions import EutilsRequestError # type: ignore -from fastapi import FastAPI +from fastapi import Depends, FastAPI from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware @@ -20,11 +20,14 @@ ) from mavedb import __version__ +from mavedb.db.session import engine +from mavedb.db.timeouts import SLOW_STATEMENT_SECONDS, apply_api_timeouts, observe_statements from mavedb.lib.exceptions import ( AmbiguousIdentifierError, HGNCGeneNotFoundError, HGNCServiceError, MixedTargetError, + MondoServiceError, NonexistentIdentifierError, ) from mavedb.lib.logging.canonical import log_request @@ -37,13 +40,17 @@ from mavedb.lib.middleware import CatchAllErrorMiddleware from mavedb.lib.permissions.exceptions import PermissionException from mavedb.lib.slack import send_slack_error +from mavedb.lib.urn_redirects import forward_retired_urns from mavedb.models import * # noqa: F403 from mavedb.routers import ( access_keys, + alleles, alphafold, api_information, + clingen_alleles, collections, controlled_keywords, + diseases, doi_identifiers, experiment_sets, experiments, @@ -75,7 +82,12 @@ # an instance of the related class has been created. configure_mappers() -app = FastAPI() +apply_api_timeouts(engine) +observe_statements(engine, SLOW_STATEMENT_SECONDS) + +# forward_retired_urns is applied to every route, so that one implementation forwards a read of a URN +# publication has retired wherever that URN points: a record, or any of its sub-resources. +app = FastAPI(dependencies=[Depends(forward_retired_urns)]) # `add_middleware` inserts at the head of the stack, so the *first* call here is the innermost layer. # CatchAllErrorMiddleware must sit inside both CORSMiddleware and the context middleware: CORS has to # decorate the 500 it produces, and the correlation id it returns comes from the context. @@ -97,9 +109,12 @@ ) app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5) app.include_router(access_keys.router) +app.include_router(alleles.router) app.include_router(api_information.router) +app.include_router(clingen_alleles.router) app.include_router(collections.router) app.include_router(controlled_keywords.router) +app.include_router(diseases.router) app.include_router(doi_identifiers.router) app.include_router(experiment_sets.router) app.include_router(experiments.router) @@ -178,6 +193,14 @@ async def hgnc_service_error_exception_handler(request: Request, exc: HGNCServic return response +@app.exception_handler(MondoServiceError) +async def mondo_service_error_exception_handler(request: Request, exc: MondoServiceError): + response = JSONResponse(status_code=503, content={"message": "Disease ontology service temporarily unavailable"}) + save_to_logging_context(format_raised_exception_info_as_dict(exc)) + log_request(request, response, time.time_ns()) + return response + + @app.exception_handler(EutilsRequestError) async def nonexistent_pmid_error_exception_handler(request: Request, exc: EutilsRequestError): response = JSONResponse(status_code=404, content={"message": str(exc)}) @@ -245,9 +268,12 @@ def customize_openapi_schema(): openapi_schema["tags"] = [ access_keys.metadata, + alleles.metadata, api_information.metadata, + clingen_alleles.metadata, collections.metadata, controlled_keywords.metadata, + diseases.metadata, doi_identifiers.metadata, experiment_sets.metadata, experiments.metadata, @@ -255,7 +281,6 @@ def customize_openapi_schema(): hgvs.metadata, licenses.metadata, # log.metadata, - mapped_variant.metadata, orcid.metadata, permissions.metadata, publication_identifiers.metadata, diff --git a/src/mavedb/view_models/allele_annotation.py b/src/mavedb/view_models/allele_annotation.py new file mode 100644 index 000000000..995d16088 --- /dev/null +++ b/src/mavedb/view_models/allele_annotation.py @@ -0,0 +1,60 @@ +"""Response view models for an allele's external annotations (VEP / gnomAD / ClinVar). + +The pydantic serialization boundary over the ``lib.allele_annotations`` transit dataclasses +(``from_attributes`` coerces them directly). These are the sparse, digest-keyed annotation blocks +that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and alongside the cross-layer +equivalence class on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather +than in either one's file. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class VepAnnotation(BaseModel): + """VEP molecular consequence for an allele and the Ensembl release it resolved under.""" + + consequence: Optional[str] = None + source_version: str + + class Config: + from_attributes = True + + +class GnomadAnnotation(BaseModel): + """gnomAD population frequency for an allele.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] = None + db_version: str + db_identifier: str + + class Config: + from_attributes = True + + +class ClinvarAnnotation(BaseModel): + """One ClinVar assertion for an allele (an allele may carry one per release).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] = None + clinvar_allele_id: str + db_version: str + + class Config: + from_attributes = True + + +class AlleleAnnotations(BaseModel): + """The external annotations for one allele, sparse — each source absent unless it has data.""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = [] + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_detail.py b/src/mavedb/view_models/allele_detail.py new file mode 100644 index 000000000..7a6fabe5b --- /dev/null +++ b/src/mavedb/view_models/allele_detail.py @@ -0,0 +1,32 @@ +"""Response view models for the allele-detail view (``GET /alleles/{digest|CAID}``). + +Pydantic serialization boundary over the :mod:`lib.allele_detail` transit dataclasses; the +allele-grain counterpart of ``view_models.variant_detail``. +""" + +from typing import Any, Optional + +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity +from mavedb.view_models.base.base import BaseModel + + +class AlleleDetail(BaseModel): + """The allele-detail envelope (``GET /alleles/{digest|CAID}``). + + ``alleles`` is the full cross-layer equivalence class, keyed by VRS digest; ``isFocus`` marks + the queried allele. ``annotations`` shares those same keys. Measurement-agnostic: no score, + classification, or re-anchored Cat-VRS (those belong to ``GET /variants/{urn}``). + """ + + digest: str + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + vrs: Optional[dict[str, Any]] = None + + alleles: dict[str, AlleleIdentity] = {} + annotations: dict[str, AlleleAnnotations] = {} + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_identity.py b/src/mavedb/view_models/allele_identity.py new file mode 100644 index 000000000..a7faf24fc --- /dev/null +++ b/src/mavedb/view_models/allele_identity.py @@ -0,0 +1,30 @@ +"""Response view model for the shared allele molecular-identity shape. + +Pydantic boundary over :class:`lib.allele_identity.AlleleIdentity`. Shared by the ``alleles`` map +on both ``GET /variants/{urn}`` and ``GET /alleles/{digest}``. See the lib docstring for the +focus-relative semantics of each axis. +""" + +from typing import Optional + +from mavedb.lib.allele_identity import AlleleDerivation +from mavedb.view_models.base.base import BaseModel + + +class AlleleIdentity(BaseModel): + """One allele in a view's ``alleles`` map, keyed by VRS digest and labelled relative to the + view's focus allele. ``isFocus`` marks the anchor; ``relation`` and ``derivation`` describe + every other member's relationship to it and are absent on the focus itself. + """ + + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + is_focus: bool + relation: Optional[str] = None + derivation: Optional[AlleleDerivation] = None + # VRS digest of this allele's c<->g projection counterpart, when one exists. + projection_of: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py new file mode 100644 index 000000000..b8f3e969d --- /dev/null +++ b/src/mavedb/view_models/allele_measurement.py @@ -0,0 +1,46 @@ +"""Response view model for the variant page's measurements list +(``GET /clingen-alleles/{caid}/measurements``). + +The pydantic serialization boundary over the ``lib.allele_measurements`` transit dataclass +(``from_attributes`` coerces it directly); aliases camelize per the shared base config. +""" + +from typing import Optional + +from mavedb.models.enums.measurement_relationship import MeasurementRelationship +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class AlleleMeasurement(BaseModel): + """One measurement in the queried ClinGen allele's cross-layer equivalence class. + + ``assayLevel`` is the level at which this measurement was actually assayed (``protein`` / ``cdna`` / + ``genomic``) — always shown, since the measured level is the clinically load-bearing fact. + ``relationship`` says how the measurement relates to the queried ClinGen id: ``direct`` (assayed at + this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or + ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``classification`` is the readable + functional classification for the variant (primary-first cascade), omitted when absent or gated. It comes + from a research-use-only calibration only when the request sets ``include_research_use_only`` and the score + set has no readable non-RUO calibration; ``classificationIsResearchUseOnly`` then marks it, and it must not + be treated as clinical evidence. Ranking uses non-RUO calls alone. ``isCurrent`` / + ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) + self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. + """ + + variant_urn: str + score: Optional[float] = None + assay_level: Optional[SequenceLevel] = None + relationship: MeasurementRelationship + assay_level_hgvs: Optional[str] = None + submitted_hgvs: Optional[str] = None + score_set_urn: str + score_set_title: str + classification: Optional[SavedFunctionalClassification] = None + classification_is_research_use_only: bool = False + is_current: bool + superseded_by_score_set: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/calibration_control.py b/src/mavedb/view_models/calibration_control.py new file mode 100644 index 000000000..9d1871ba6 --- /dev/null +++ b/src/mavedb/view_models/calibration_control.py @@ -0,0 +1,89 @@ +"""Pydantic view models for calibration controls. + +A calibration control pairs a MaveDB variant with its independently known clinical +significance, serving as empirical ground truth when a calibration's score thresholds +are derived. These models are the API-layer representation used when controls are +created, updated, and served through the calibration endpoints. +""" + +from datetime import date +from typing import Any, Optional + +from pydantic import model_validator + +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.view_models import record_type_validator, set_record_type +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.user import SavedUser + + +class CalibrationControlBase(BaseModel): + """Fields shared by every calibration control view model. + + ``variant_urn`` is the external identifier for the control's variant; the internal + primary key is never exposed. + """ + + variant_urn: str + clinical_status: CalibrationControlStatus + + +class CalibrationControlModify(CalibrationControlBase): + """Model used to modify an existing calibration control. + + Carries no additional fields — only the base fields can be updated. + """ + + pass + + +class CalibrationControlCreate(CalibrationControlModify): + """Model used to create a calibration control. + + Carries no additional fields — only the modifiable fields are required for creation. + """ + + pass + + +class SavedCalibrationControl(CalibrationControlBase): + """Persisted calibration control, including identifier and audit metadata.""" + + record_type: str = None # type: ignore + _record_type_factory = record_type_validator()(set_record_type) + + id: int + # The calibration's own functional classification (range or class) which contains this control's + # variant, or None when the variant lands under none of them. + functional_classification_id: Optional[int] = None + creation_date: date + modification_date: date + created_by: SavedUser + modified_by: SavedUser + + class Config: + """Pydantic configuration (ORM mode).""" + + from_attributes = True + + @model_validator(mode="before") + def generate_variant_urn(cls, data: Any): + """Expose the control's variant as ``variant_urn`` when building from an ORM object. + + The ORM row references a ``Variant`` relationship, not a bare URN. Mirrors the + synthetic-field pattern used by ``ScoreCalibrationWithScoreSetUrn``. + """ + if hasattr(data, "variant"): + try: + data.__setattr__("variant_urn", data.variant.urn) + except (AttributeError, KeyError) as exc: + raise ValidationError(f"Unable to coerce variant urn for {cls.__name__}: {exc}.") # type: ignore + + return data + + +class CalibrationControl(SavedCalibrationControl): + """Calibration control with its associated calibration and score set URNs.""" + + pass diff --git a/src/mavedb/view_models/clinical_control.py b/src/mavedb/view_models/clinical_control.py index 85cd78348..7e6a17d06 100644 --- a/src/mavedb/view_models/clinical_control.py +++ b/src/mavedb/view_models/clinical_control.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariantCreate, MappedVariantForClinicalControl + from mavedb.view_models.mapped_variant import MappedVariantCreate class ClinicalControlBase(BaseModel): @@ -41,8 +41,17 @@ class Config: from_attributes = True -class SavedClinicalControlWithMappedVariants(SavedClinicalControl): - mapped_variants: Sequence["MappedVariantForClinicalControl"] +class ClinvarVariantLink(BaseModel): + """One score-set variant a ClinVar control reaches, tagged with the annotated allele's digest.""" + + variant_urn: str + # VRS digest of the allele this control annotates. Many controls + # may share the same variant_urn, but each will have a different allele_digest. + allele_digest: Optional[str] = None + + +class SavedClinicalControlWithClinvarLinks(SavedClinicalControl): + clinvar_links: Sequence[ClinvarVariantLink] # Properties to return to non-admin clients @@ -50,7 +59,7 @@ class ClinicalControl(SavedClinicalControl): pass -class ClinicalControlWithMappedVariants(SavedClinicalControlWithMappedVariants): +class ClinicalControlWithClinvarLinks(SavedClinicalControlWithClinvarLinks): pass diff --git a/src/mavedb/view_models/csv_namespace.py b/src/mavedb/view_models/csv_namespace.py index e4048cf2e..3a868ecf1 100644 --- a/src/mavedb/view_models/csv_namespace.py +++ b/src/mavedb/view_models/csv_namespace.py @@ -30,6 +30,9 @@ class AvailableCsvNamespace(BaseModel): A picker should group calibrations by this when a response spans more than one score set. """ + research_use_only: bool = False + """Whether this is a research-use-only calibration, for a picker to tag beside the label.""" + selected_by_default: bool = True """Whether a picker should open with this group checked. diff --git a/src/mavedb/view_models/gnomad_variant.py b/src/mavedb/view_models/gnomad_variant.py index 97dd675e6..116a4e21b 100644 --- a/src/mavedb/view_models/gnomad_variant.py +++ b/src/mavedb/view_models/gnomad_variant.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariant, MappedVariantCreate, SavedMappedVariant + from mavedb.view_models.mapped_variant import MappedVariantCreate class GnomADVariantBase(BaseModel): @@ -54,10 +54,21 @@ class Config: from_attributes = True -class SavedGnomADVariantWithMappedVariants(SavedGnomADVariant): - """Saved GnomAD variant records with mapped variants.""" +class GnomadVariantLink(BaseModel): + """One score-set variant a gnomAD frequency record reaches, tagged with the annotated allele's digest. - mapped_variants: Sequence["SavedMappedVariant"] + Mirrors :class:`clinical_control.ClinvarVariantLink`: a gnomAD variant fans out to every allele that + resolved to it, and each allele belongs to a score-set variant. + """ + + variant_urn: str + allele_digest: Optional[str] = None + + +class SavedGnomADVariantWithVariantLinks(SavedGnomADVariant): + """Saved gnomAD variant paired with the score-set variants (and annotated allele digests) it links to.""" + + variant_links: Sequence[GnomadVariantLink] class GnomADVariant(SavedGnomADVariant): @@ -66,7 +77,7 @@ class GnomADVariant(SavedGnomADVariant): pass -class GnomADVariantWithMappedVariants(SavedGnomADVariantWithMappedVariants): - """GnomAD variant view model with mapped variants for non-admin clients.""" +class GnomADVariantWithVariantLinks(SavedGnomADVariantWithVariantLinks): + """GnomAD variant + its score-set variant links, for non-admin clients.""" - mapped_variants: Sequence["MappedVariant"] + pass diff --git a/src/mavedb/view_models/hgvs.py b/src/mavedb/view_models/hgvs.py new file mode 100644 index 000000000..e295cc9df --- /dev/null +++ b/src/mavedb/view_models/hgvs.py @@ -0,0 +1,5 @@ +from mavedb.view_models.base.base import BaseModel + + +class HgvsValidationRequest(BaseModel): + variant: str diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py new file mode 100644 index 000000000..1d3a2bf1e --- /dev/null +++ b/src/mavedb/view_models/lean_variant.py @@ -0,0 +1,74 @@ +"""Response view models for the lean whole-set view (``GET /score-sets/{urn}/variants``). + +The pydantic serialization boundary over the ``lib.score_set_variants`` transit dataclasses. ``from_attributes`` +lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` +coerce them (field names line up); aliases camelize per the shared base config, so clients see +``variantUrn`` / ``assayLevel`` / ``mapped`` etc. ``response_model_exclude_none`` drops absent fields, so a +slot with no parsed block serializes as just ``{"hgvs": "..."}`` and a null mapped level drops out of the +``mapped`` object entirely. +""" + +from typing import Optional + +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.view_models.base.base import BaseModel + + +class HgvsField(BaseModel): + """An HGVS expression with its parsed substitution block riding alongside when representable. + + ``hgvs`` is always present; ``position``/``ref``/``alt`` appear only for a placeable simple + substitution (the heatmap grid) and are omitted for splice/indels/multivariants. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + class Config: + from_attributes = True + + +class MappedTriple(BaseModel): + """The mapped (reference-frame) HGVS keyed by level — the canonical projection of the measured change. + + One slot per level. A nucleotide assay populates the measured slot (``mapped[assayLevel]``) and the + projections it has (``cdna`` is the level-invariant search key, present even when ``assayLevel`` is + ``genomic``; an intronic or UTR change has no ``protein``); a protein assay populates only ``protein`` + (the ambiguous c/g fan-out is not fabricated). Null slots are + omitted under ``response_model_exclude_none``. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + class Config: + from_attributes = True + + +class LeanVariant(BaseModel): + """One pre-chewed per-variant record feeding the score-set table, heatmap, and histograms. + + ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed + annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) carry + the depositor's frame for the heatmap's raw↔mapped toggle. The mapped (reference) frame is the + ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``SequenceLevel`` value) naming + the measured/canonical slot: ``mapped[assayLevel]`` is the measured representation and ``mapped.cdna`` + the level-invariant search key. Fields are omitted when null. + """ + + variant_urn: str + score: Optional[float] = None + consequence: Optional[str] = None + clingen_allele_id: Optional[str] = None + assay_level_digest: Optional[str] = None + hgvs_nt: Optional[HgvsField] = None + hgvs_pro: Optional[HgvsField] = None + hgvs_splice: Optional[HgvsField] = None + assay_level: Optional[SequenceLevel] = None + mapped: MappedTriple = MappedTriple() + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/mapped_variant.py b/src/mavedb/view_models/mapped_variant.py index 2d4856c88..7c03fd103 100644 --- a/src/mavedb/view_models/mapped_variant.py +++ b/src/mavedb/view_models/mapped_variant.py @@ -7,7 +7,7 @@ from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel from mavedb.view_models.target_gene_mapping import SavedTargetGeneMapping, TargetGeneMapping @@ -28,7 +28,7 @@ class MappedVariantBase(BaseModel): current: bool # Per-mapping QC annotations/warnings. - alignment_level: Optional[AnnotationLayer] = None + alignment_level: Optional[SequenceLevel] = None at_mismatched_locus: Optional[bool] = None near_gap: Optional[bool] = None @@ -111,22 +111,6 @@ class MappedVariantWithControls(SavedMappedVariantWithControls): gnomad_variants: Sequence["GnomADVariant"] -class MappedVariantForClinicalControl(BaseModel): - variant_urn: str - - class Config: - from_attributes = True - - @model_validator(mode="before") - def generate_score_set_urn_list(cls, data: Any): - if not hasattr(data, "variant_urn") and hasattr(data, "variant"): - try: - data.__setattr__("variant_urn", None if not data.variant else data.variant.urn) - except AttributeError as exc: - raise ValidationError(f"Unable to create {cls.__name__} without attribute: {exc}.") # type: ignore - return data - - # ruff: noqa: E402 from mavedb.view_models.clinical_control import ClinicalControl, ClinicalControlBase, SavedClinicalControl from mavedb.view_models.gnomad_variant import GnomADVariant, GnomADVariantBase, SavedGnomADVariant diff --git a/src/mavedb/view_models/score_calibration.py b/src/mavedb/view_models/score_calibration.py index 857d4e49f..5d5e6337e 100644 --- a/src/mavedb/view_models/score_calibration.py +++ b/src/mavedb/view_models/score_calibration.py @@ -7,8 +7,10 @@ from datetime import date from typing import TYPE_CHECKING, Any, Collection, Optional, Sequence, Union +from ga4gh.core.models import MappableConcept from pydantic import Field, field_validator, model_validator +from mavedb.lib.mondo import generic_disease_mappable_concept, mondo_term_to_mappable_concept from mavedb.lib.oddspaths import oddspaths_evidence_strength_equivalent from mavedb.lib.validation.exceptions import ValidationError from mavedb.lib.validation.transform import ( @@ -26,6 +28,10 @@ SavedACMGClassification, ) from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.calibration_control import ( + CalibrationControlCreate, + SavedCalibrationControl, +) from mavedb.view_models.publication_identifier import ( PublicationIdentifier, PublicationIdentifierBase, @@ -282,6 +288,8 @@ class ScoreCalibrationBase(BaseModel): baseline_score_description: Optional[str] = None notes: Optional[str] = None + controls_not_phi: Optional[bool] = None + functional_classifications: Optional[Sequence[FunctionalClassificationBase]] = None threshold_sources: Sequence[PublicationIdentifierBase] evidence_sources: Sequence[PublicationIdentifierBase] @@ -432,6 +440,15 @@ class ScoreCalibrationModify(ScoreCalibrationBase): score_set_urn: Optional[str] = None functional_classifications: Optional[Sequence[FunctionalClassificationModify]] = None + # None means "no change" on modify; an empty list clears all controls. + controls: Optional[Sequence[CalibrationControlCreate]] = None + disease: Optional[str] = Field( + None, + description=( + 'The MONDO code (e.g. "MONDO:0015263") for this calibration\'s disease context; validated ' + 'against OLS. Omitted or null resolves to the generic "disease or disorder" term.' + ), + ) threshold_sources: Sequence[PublicationIdentifierCreate] evidence_sources: Sequence[PublicationIdentifierCreate] method_sources: Sequence[PublicationIdentifierCreate] @@ -498,6 +515,12 @@ class SavedScoreCalibration(ScoreCalibrationBase): private: bool = True functional_classifications: Optional[Sequence[SavedFunctionalClassification]] = None + # The full controls list lives on the detail models; the base (and any list/collection response) + # carries only the count to keep those payloads small. See ScoreCalibrationDetailWithScoreSetUrn. + controls_count: int = 0 + # The FK is non-nullable, so this is always populated by ``generate_disease_concept`` when building + # from an ORM object; an unspecified disease resolves to the generic "disease or disorder" concept. + disease: MappableConcept threshold_sources: Sequence[SavedPublicationIdentifier] evidence_sources: Sequence[SavedPublicationIdentifier] method_sources: Sequence[SavedPublicationIdentifier] @@ -562,11 +585,27 @@ def generate_threshold_classification_and_method_sources(cls, data: Any): # typ ) return data + @model_validator(mode="before") + def generate_disease_concept(cls, data: Any): + """Serialize the stored MONDO term as a ``disease`` MappableConcept when building from an ORM object. + + Falls back to the generic concept if the relationship is unexpectedly empty, so the non-nullable + field is always satisfied. + """ + if hasattr(data, "disease_term"): + term = data.disease_term + data.__setattr__( + "disease", + mondo_term_to_mappable_concept(term) if term is not None else generic_disease_mappable_concept(), + ) + return data + class ScoreCalibration(SavedScoreCalibration): """Complete score calibration model returned by the API.""" functional_classifications: Optional[Sequence[FunctionalClassification]] = None + controls: Sequence[SavedCalibrationControl] = [] threshold_sources: Sequence[PublicationIdentifier] evidence_sources: Sequence[PublicationIdentifier] method_sources: Sequence[PublicationIdentifier] @@ -575,7 +614,11 @@ class ScoreCalibration(SavedScoreCalibration): class ScoreCalibrationWithScoreSetUrn(SavedScoreCalibration): - """Complete score calibration model returned by the API, with score_set_urn.""" + """Score calibration model with score_set_urn, used for list/collection responses. + + Carries ``controls_count`` (from the base) but not the full controls list — see + ``ScoreCalibrationDetailWithScoreSetUrn`` for the single-item detail representation. + """ score_set_urn: str @@ -589,3 +632,9 @@ def generate_score_set_urn(cls, data: Any): f"Unable to coerce score set urn for {cls.__name__}: {exc}." # type: ignore ) return data + + +class ScoreCalibrationDetailWithScoreSetUrn(ScoreCalibrationWithScoreSetUrn): + """Single-calibration detail response: adds the full controls list to the list model.""" + + controls: Sequence[SavedCalibrationControl] = [] diff --git a/src/mavedb/view_models/score_distribution.py b/src/mavedb/view_models/score_distribution.py new file mode 100644 index 000000000..b4272b18e --- /dev/null +++ b/src/mavedb/view_models/score_distribution.py @@ -0,0 +1,15 @@ +from mavedb.view_models.base.base import BaseModel + + +class ScoreDistribution(BaseModel): + """Binned summary of a score set's scores, built by ``mavedb.lib.score_distribution.summarize_scores``. + + ``counts`` holds equal-width bins spanning ``[min, max]``; scores that are null or not finite are + counted in ``null_count`` instead. + """ + + version: int + min: float + max: float + counts: list[int] + null_count: int diff --git a/src/mavedb/view_models/score_set.py b/src/mavedb/view_models/score_set.py index ce7a0730a..c10a5353f 100644 --- a/src/mavedb/view_models/score_set.py +++ b/src/mavedb/view_models/score_set.py @@ -38,6 +38,7 @@ ScoreCalibration, ScoreCalibrationCreate, ) +from mavedb.view_models.score_distribution import ScoreDistribution from mavedb.view_models.score_set_dataset_columns import DatasetColumns, SavedDatasetColumns from mavedb.view_models.target_gene import ( SavedTargetGene, @@ -439,6 +440,7 @@ class ScoreSet(SavedScoreSet): mapping_errors: Optional[dict] = None score_calibrations: Optional[Sequence[ScoreCalibration]] = None # type: ignore[assignment] dataset_columns: Optional[DatasetColumns] = None # type: ignore[assignment] + score_distribution: Optional[ScoreDistribution] = None class ScoreSetWithVariants(ScoreSet): diff --git a/src/mavedb/view_models/target_gene_mapping.py b/src/mavedb/view_models/target_gene_mapping.py index 13e64d1a7..535508a42 100644 --- a/src/mavedb/view_models/target_gene_mapping.py +++ b/src/mavedb/view_models/target_gene_mapping.py @@ -7,13 +7,13 @@ from datetime import date from typing import Any, Optional -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel class TargetGeneMappingBase(BaseModel): - alignment_level: AnnotationLayer + alignment_level: SequenceLevel preferred: bool = False reference_assembly: Optional[str] = None diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index b01b01832..d9257da37 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -1,15 +1,26 @@ from datetime import date -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel -from mavedb.view_models.mapped_variant import MappedVariant, SavedMappedVariant +from mavedb.view_models.mapped_variant import SavedMappedVariant -if TYPE_CHECKING: - from mavedb.view_models.score_set import ScoreSet, ShortScoreSet + +class VariantVrsMatch(BaseModel): + """One variant the queried VRS identifier identifies, before or after mapping. + + The ``GET /variants/vrs/{identifier}`` lookup result. The ClinGen id, VRS id and level ride from the + matched (deduplicated) allele for a post-mapped match, or from the variant's measured allele for a + pre-mapped match; they are null when that variant did not map. + """ + + variant_urn: str + clingen_allele_id: Optional[str] = None + vrs_id: Optional[str] = None + level: Optional[str] = None class VariantEffectMeasurementBase(BaseModel): @@ -74,39 +85,3 @@ class VariantEffectMeasurement(SavedVariantEffectMeasurement): """Variant effect measurement view model returned to most clients""" pass - - -class VariantEffectMeasurementWithScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and score set""" - - score_set: "ScoreSet" - mapped_variants: list[MappedVariant] - - -class VariantEffectMeasurementWithShortScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and a limited set of score set details""" - - score_set: "ShortScoreSet" - mapped_variants: list[MappedVariant] - - -class ClingenAlleleIdVariantLookupsRequest(BaseModel): - """A request to search for variants matching a list of ClinGen allele IDs""" - - clingen_allele_ids: list[str] - - -class Variant(BaseModel): - """View model for a variant, defined by its ClinGen allele id, with associated variant effect measurements""" - - clingen_allele_id: str - variant_effect_measurements: list[VariantEffectMeasurementWithShortScoreSet] - - -class ClingenAlleleIdVariantLookupResponse(BaseModel): - """Response model for a variant lookup by ClinGen allele ID""" - - clingen_allele_id: str - exact_match: Optional[Variant] = None - equivalent_nt: list[Variant] = [] - equivalent_aa: list[Variant] = [] diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py new file mode 100644 index 000000000..2c9f6004d --- /dev/null +++ b/src/mavedb/view_models/variant_detail.py @@ -0,0 +1,84 @@ +"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``). + +The pydantic serialization boundary over the ``lib.variant_detail`` transit dataclasses +(``from_attributes`` coerces them directly); aliases camelize per the shared base config. Kept in its +own module — mirroring ``lib/variant_detail.py`` and the ``lean_variant`` precedent — so +``view_models/variant.py`` stays the core measurement + lookup file. +""" + +from typing import Any, Optional + +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.lean_variant import MappedTriple +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class VariantClassification(BaseModel): + """A functional classification the variant falls into, tagged with its calibration context. + + A score set may carry several calibrations, so a variant has one classification per calibration; + ``primary`` flags the UI default. The classifications are calibration-derived but as-of-invariant + (calibrations carry no valid-time), so they are always the current calibration state. + """ + + calibration_id: int + primary: bool + classification: SavedFunctionalClassification + + class Config: + from_attributes = True + + +class VariantDetail(BaseModel): + """The assayed variant-detail envelope (``GET /variants/{urn}``). + + Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair + is a client-side toggle, no refetch; the ``preMapped``/``postMapped`` raw VRS pair lets a + VRS/bulk consumer read the assayed-level and measured VRS directly) plus the spec-pure GA4GH + ``molecularRepresentation`` (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer + rides alongside, keyed by VRS digest: the ``alleles`` identity sidecar (per-allele ``level`` / + ``hgvs`` / ``clingenAlleleId`` / ``relation`` — one entry per linked allele, sharing keys with + ``annotations``) and the ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a + superseded variant self-describe: ``supersededByScoreSet`` is the superseding *score set*'s URN, + not a variant URN. Supersession is versioned at the score-set level, and a newer version may add, + drop, or renumber variants — so there is no stable superseding-*variant* pointer to hand back; a + consumer resolves the current measurement by looking this variant up within that score set. + + Unlike most MaveDB response models, this one serializes with ``exclude_none=False`` on both routes + that emit it (``GET /variants/{urn}`` and the bulk ``GET /score-sets/{urn}/variant-details`` NDJSON + stream): the shape is a stable, self-describing envelope, so every record carries the same key set + and an unmapped variant reads ``preMapped``/``postMapped``/``molecularRepresentation`` as ``null`` + rather than dropping them. The two routes are kept in lockstep — the same object, one shape. + """ + + urn: str + scores: Optional[dict[str, Any]] = None + counts: Optional[dict[str, Any]] = None + classifications: list[VariantClassification] = [] + + assay_level: Optional[SequenceLevel] = None + target_hgvs: Optional[str] = None + reference_hgvs: Optional[str] = None + assay_level_digest: Optional[str] = None + clingen_allele_id: Optional[str] = None + + pre_mapped: Optional[dict[str, Any]] = None + post_mapped: Optional[dict[str, Any]] = None + + molecular_representation: Optional[dict[str, Any]] = None + mode: Optional[str] = None + # The measured variant's mapped HGVS at every level it has, including levels with no allele in + # ``alleles`` (an intronic coding projection has no VRS representation). + mapped: Optional[MappedTriple] = None + alleles: dict[str, AlleleIdentity] = {} + + annotations: dict[str, AlleleAnnotations] = {} + + is_current: bool + superseded_by_score_set: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/worker/README.md b/src/mavedb/worker/README.md index 34c66ed0f..77f6328fe 100644 --- a/src/mavedb/worker/README.md +++ b/src/mavedb/worker/README.md @@ -60,16 +60,17 @@ worker/ │ ├── jobs/ # ── Business Layer ── │ ├── registry.py # Central registry of all job functions -│ ├── variant_processing/ # Variant creation and mapping jobs +│ ├── variant_processing/ # Variant creation, mapping and reverse translation jobs │ │ ├── creation.py # create_variants_for_score_set -│ │ └── mapping.py # map_variants_for_score_set +│ │ ├── mapping.py # map_variants_for_score_set +│ │ └── reverse_translation.py # reverse_translate_variants_for_score_set │ ├── external_services/ # Integration with external APIs │ │ ├── clingen.py # CAR and LDH submission +│ │ ├── clingen_cache.py # ClinGen allele cache warming │ │ ├── clinvar.py # ClinVar control refresh │ │ ├── gnomad.py # gnomAD variant linking -│ │ ├── hgvs.py # HGVS annotation │ │ ├── uniprot.py # UniProt mapping submission/polling -│ │ └── variant_translation.py # Variant translation population +│ │ └── vep.py # VEP consequence population │ ├── data_management/ # Database maintenance jobs │ │ └── views.py # Materialized view refresh │ ├── pipeline_management/ # Pipeline orchestration jobs @@ -81,6 +82,8 @@ worker/ │ └── constants.py # Job-level constants │ ├── lib/ # ── Infrastructure Layer ── +│ ├── projection.py # Measured projection groups (c./g./p. HGVS per mapping record) +│ ├── translation_ports.py # variant-annotation translation ports (coordinate translator, UTA) │ ├── decorators/ # Job/pipeline lifecycle decorators │ │ ├── job_management.py # @with_job_management │ │ ├── pipeline_management.py # @with_pipeline_management @@ -93,10 +96,10 @@ worker/ │ ├── constants.py # Status grouping constants │ ├── exceptions.py # Exception hierarchy │ ├── types.py # TypedDicts (RetryHistoryEntry, PipelineProgress) -│ └── utils.py # Dependency checking helpers, classify_exception() +│ └── utils.py # Dependency checking helpers, classify_exception(), ARQ job id/queue routing │ └── settings/ # ARQ worker configuration - ├── worker.py # ArqWorkerSettings class + ├── worker.py # ArqWorkerSettings (main) and ArqFastLaneWorkerSettings classes ├── lifecycle.py # Startup/shutdown/job hooks, standalone_ctx() ├── redis.py # Redis connection settings └── constants.py # Environment variable handling diff --git a/src/mavedb/worker/__init__.py b/src/mavedb/worker/__init__.py index 1f1489b59..04d73a604 100644 --- a/src/mavedb/worker/__init__.py +++ b/src/mavedb/worker/__init__.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import configure_mappers from mavedb.models import * # noqa: F403 -from mavedb.worker.settings import ArqWorkerSettings +from mavedb.worker.settings import ArqFastLaneWorkerSettings, ArqWorkerSettings # Scan all our model classes and create backref attributes. Otherwise, these attributes only get added to classes once # an instance of the related class has been created. Since the worker is a distinct service, we should make sure this @@ -9,5 +9,6 @@ configure_mappers() # Expose the worker settings to the arq CLI Worker initializer -global WorkerSettings +global WorkerSettings, FastLaneWorkerSettings WorkerSettings = ArqWorkerSettings +FastLaneWorkerSettings = ArqFastLaneWorkerSettings diff --git a/src/mavedb/worker/best_practices.md b/src/mavedb/worker/best_practices.md index e83eb42f1..92b672617 100644 --- a/src/mavedb/worker/best_practices.md +++ b/src/mavedb/worker/best_practices.md @@ -66,9 +66,9 @@ return JobExecutionOutcome.succeeded(data={"variants_created": count}) ### Failed — a business-logic failure (not a bug) ```python # Missing data, validation failure, precondition not met -if not mapped_variants: +if not mapping_records: return JobExecutionOutcome.failed( - reason="No mapped variants found for score set", + reason="No mapping records found for score set", data={"score_set_id": score_set_id} ) ``` @@ -120,6 +120,14 @@ job_manager.update_progress(50, 100, "Writing to database.") job_manager.update_progress(100, 100, "Variant creation complete.") ``` +### Long external delegations + +A job that hands work to an opaque external call — a single blocking request whose duration you don't control and whose interior you can't see — breaks the checkpoint model. `cleanup_stalled_jobs` reaps a RUNNING job whose `progress_updated_at` hasn't advanced in `PROGRESS_STALL_MINUTES` (30), but you cannot emit real progress while parked inside a black-box call. The heartbeat conflates two signals that opaque delegation forces apart: **liveness** (is the worker alive and still executing — always answerable) and **progress** (is the work advancing or wedged — not answerable from your side of the call). + +**Rule: any long external delegation must expose progress, or be given a bounded duration at its boundary.** Absent one of those, stall detection degrades to liveness and "the worker is up" becomes the only thing you can assert. Real progress comes only from the delegate reporting it (streaming / callback / a status you poll) or from chunking the call so each unit is a real checkpoint — both require a change on the delegate's side and are the preferred fix. + +Until then, the interim pattern (see `map_variants_for_score_set`) is a **liveness keepalive** — a concurrent `asyncio` task that refreshes the heartbeat on a cadence well under `PROGRESS_STALL_MINUTES` so a live worker is not falsely reaped — paired with a **size-aware `asyncio.wait_for` budget** so a genuinely wedged delegate fails in a sensible window rather than riding the 23.5h wall-clock backstop. The keepalive dies with a crashed worker, so real crashes still go stale and get reaped. This is an explicit workaround; it does not claim progress it cannot see. Never `time.sleep` in the keepalive — it blocks the shared event loop and freezes every other job on the worker. + ## Logging Context Always set up logging context early in the job function: @@ -162,8 +170,11 @@ async def submit_to_external_service(ctx: dict, job_id: int, job_manager: JobMan ).one() # 3. Check preconditions - if not score_set.mapped_variants: - return JobExecutionOutcome.failed(reason="No mapped variants to submit") + has_mappings = job_manager.db.scalar( + select(exists().where(MappingRecord.score_set_id == score_set.id, MappingRecord.current)) + ) + if not has_mappings: + return JobExecutionOutcome.failed(reason="No mapping records to submit") # 4. Submit to the service (let exceptions propagate for service errors) result = await external_client.submit(score_set) diff --git a/src/mavedb/worker/job_registry.md b/src/mavedb/worker/job_registry.md index 952b4359d..3b7deb5d9 100644 --- a/src/mavedb/worker/job_registry.md +++ b/src/mavedb/worker/job_registry.md @@ -13,6 +13,7 @@ BACKGROUND_FUNCTIONS: List[Callable] = [ # Variant processing jobs create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, # External service jobs submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -42,8 +43,8 @@ BACKGROUND_CRONJOBS: List[CronJob] = [ cron( cleanup_stalled_jobs, name="cleanup_stalled_jobs_cron", - minute={15, 45}, # Every 30 minutes - keep_result=timedelta(minutes=25).total_seconds(), + minute={5, 15, 25, 35, 45, 55}, # Every 10 minutes + keep_result=timedelta(minutes=9).total_seconds(), ), ] ``` @@ -239,9 +240,19 @@ class ArqWorkerSettings: redis_settings = RedisWorkerSettings functions = BACKGROUND_FUNCTIONS cron_jobs = BACKGROUND_CRONJOBS - job_timeout = 5 * 60 * 60 # 5 hours + max_jobs = MAX_JOBS # 2; limits event-loop starvation from sync DB calls + job_timeout = JOB_TIMEOUT_SECONDS # 24 hours + allow_abort_jobs = True # Lets cleanup_stalled_jobs abort a wedged RUNNING job ``` +### Fast-lane worker + +`ArqFastLaneWorkerSettings` runs as a second worker process (`arq mavedb.worker.FastLaneWorkerSettings`, its own container on the worker host). It consumes only `FAST_LANE_QUEUE_NAME` and registers only the jobs in `FAST_LANE_JOB_FUNCTIONS` (`start_pipeline`, `create_variants_for_score_set`), so new pipelines get their variants created even while hour-long mapping jobs hold every main-worker slot. It runs no cron jobs. + +Routing lives in `arq_queue_name()` (`lib/managers/utils.py`). Every ARQ call that takes a queue — `enqueue_job(_queue_name=...)`, `Job(..., _queue_name=...).status()` / `.abort()` — must resolve it from the `JobRun` there; a status check against the wrong queue reports a queued job as missing. To move a job into the fast lane, add its function name to `FAST_LANE_JOB_FUNCTIONS`; keep it short-running, since it shares two slots with variant creation. + +ARQ reads settings from the class `__dict__`, so the fast-lane class repeats its attributes rather than subclassing `ArqWorkerSettings`. + The lifecycle hooks (in `settings/lifecycle.py`) manage the ARQ context dict (`ctx`): - `startup`: Creates `ProcessPoolExecutor` for CPU-intensive tasks - `on_job_start`: Initializes `hdp` (CDOT REST data provider) and `state` dict diff --git a/src/mavedb/worker/jobs/__init__.py b/src/mavedb/worker/jobs/__init__.py index e421bbad2..488398117 100644 --- a/src/mavedb/worker/jobs/__init__.py +++ b/src/mavedb/worker/jobs/__init__.py @@ -33,11 +33,13 @@ from mavedb.worker.jobs.variant_processing.mapping import ( map_variants_for_score_set, ) +from mavedb.worker.jobs.variant_processing.reverse_translation import reverse_translate_variants_for_score_set __all__ = [ # Variant processing jobs "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", # External service integration jobs "submit_score_set_mappings_to_car", "submit_score_set_mappings_to_ldh", diff --git a/src/mavedb/worker/jobs/external_services/__init__.py b/src/mavedb/worker/jobs/external_services/__init__.py index 4537c0edd..78af92516 100644 --- a/src/mavedb/worker/jobs/external_services/__init__.py +++ b/src/mavedb/worker/jobs/external_services/__init__.py @@ -5,8 +5,6 @@ - ClinGen cache pre-warming to prevent stampede on downstream annotation jobs - UniProt for protein sequence annotation and ID mapping - gnomAD for population frequency and genomic context data -- HGVS for standardized variant nomenclature population -- Variant Translation for PA<->CA allele relationship mapping - VEP for functional consequence annotation """ @@ -18,12 +16,10 @@ from .clingen_cache import warm_clingen_cache from .clinvar import refresh_clinvar_controls from .gnomad import link_gnomad_variants -from .hgvs import populate_hgvs_for_score_set from .uniprot import ( poll_uniprot_mapping_jobs_for_score_set, submit_uniprot_mapping_jobs_for_score_set, ) -from .variant_translation import populate_variant_translations_for_score_set from .vep import populate_vep_for_score_set __all__ = [ @@ -32,8 +28,6 @@ "warm_clingen_cache", "refresh_clinvar_controls", "link_gnomad_variants", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "populate_vep_for_score_set", diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 639160ed5..3b85f60c8 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -3,7 +3,6 @@ This module contains jobs for submitting mapped variants to ClinGen services: - ClinGen Allele Registry (CAR) for allele registration - ClinGen Linked Data Hub (LDH) for data submission -- Variant linking and association management These jobs enable integration with the ClinGen ecosystem for clinical variant interpretation and data sharing. @@ -12,13 +11,19 @@ import asyncio import functools import logging +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any from sqlalchemy import select +from sqlalchemy.orm import Session from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import ( CAR_SUBMISSION_ENDPOINT, CLIN_GEN_SUBMISSION_ENABLED, + DEFAULT_CAR_SUBMISSION_BATCH_SIZE, DEFAULT_LDH_SUBMISSION_BATCH_SIZE, LDH_SUBMISSION_ENDPOINT, ) @@ -27,11 +32,19 @@ ClinGenAlleleRegistryService, ClinGenLdhService, ) +from mavedb.lib.types.clingen import ClinGenAllele from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.utils import batched from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele as AlleleModel +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params @@ -41,6 +54,60 @@ logger = logging.getLogger(__name__) +@dataclass +class _AlleleEntry: + post_mapped: dict | None + existing_caid: str | None + + +def _response_hgvs(response: ClinGenAllele) -> set[str]: + """Every HGVS expression a CAR allele response lists, across genomic, transcript and protein alleles.""" + definitions = [ + *response.get("genomicAlleles", []), + *response.get("transcriptAlleles", []), + *response.get("aminoAcidAlleles", []), + ] + return {hgvs for definition in definitions for hgvs in definition.get("hgvs", [])} + + +def _retire_caid_keyed_links(db: Session, allele_id: int) -> tuple[int, int]: + """Retire an allele's live gnomAD and ClinVar links, which were matched through its CAID. + + Both linkers skip an allele only while it holds a live link, so retiring them makes the next + gnomAD and ClinVar runs link the allele again under its corrected CAID. + """ + gnomad = db.execute(GnomadAlleleLink.retire_live_where(GnomadAlleleLink.allele_id == allele_id)) + clinvar = db.execute(ClinvarAlleleLink.retire_live_where(ClinvarAlleleLink.allele_id == allele_id)) + return gnomad.rowcount, clinvar.rowcount # type: ignore[attr-defined] + + +def _annotate_caid( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CLINGEN_ALLELE_ID event for an allele (the CAID is an allele-level fact). + + The single choke point for CAR's status writes. Provenance (which variants drove the + registration) is derived from the live links as-of the event, not fanned out here. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele_id, + disposition=disposition, + reason=reason, + metadata=meta or None, + ) + + @with_pipeline_management async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ @@ -53,16 +120,22 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: - score_set_id (int): ID of the ScoreSet to process - correlation_id (str): Correlation ID for tracking + Optional job_params: + - force_reregister (bool): resubmit alleles that already hold a CAID. Where CAR assigns a different + one, it replaces the stored CAID (recorded as ``corrected``) and the allele's gnomAD and ClinVar + links are retired. + Args: ctx (dict): Worker context containing DB and Redis connections job_manager (JobManager): Manager for job lifecycle and DB operations Side Effects: - - Updates MappedVariant records with ClinGen Allele IDs + - Updates Allele records with ClinGen Allele IDs - Submits data to ClinGen Allele Registry + - Retires the gnomAD and ClinVar links of an allele whose CAID is corrected Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele counts (submitted/registered/already-registered/failed). """ # Get the job definition we are working on job = job_manager.get_job() @@ -70,11 +143,12 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: _job_required_params = ["score_set_id", "correlation_id"] validate_job_params(_job_required_params, job) - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + # A pipeline param may arrive as a string from --extra-param, where bool("false") would be True. + raw_force = job.job_params.get("force_reregister", False) # type: ignore[union-attr] + force_reregister = raw_force is True or str(raw_force).lower() == "true" - # Setup initial context and progress job_manager.save_to_context( { "application": "mavedb-worker", @@ -86,7 +160,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) - # Ensure we've enabled ClinGen submission if not CLIN_GEN_SUBMISSION_ENABLED: logger.warning( msg="ClinGen submission is disabled via configuration, skipping submission of mapped variants to CAR.", @@ -95,7 +168,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.skipped(data={"reason": "ClinGen submission disabled"}) - # Check for CAR submission endpoint if not CAR_SUBMISSION_ENDPOINT: logger.warning( msg="ClinGen Allele Registry submission is disabled (no submission endpoint), unable to complete submission of mapped variants to CAR.", @@ -107,180 +179,352 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: failure_category=FailureCategory.CONFIGURATION_ERROR, ) - # Fetch mapped variants with post-mapped data for the score set - variant_post_mapped_objects = job_manager.db.execute( - select(MappedVariant.id, MappedVariant.post_mapped) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) - ).all() - - # Track total variants to submit - job_manager.save_to_context({"total_variants_to_submit_car": len(variant_post_mapped_objects)}) - if not variant_post_mapped_objects: + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + job_manager.save_to_context({"total_allele_variant_pairs": len(allele_rows)}) + if not allele_rows: logger.warning( - msg="No current mapped variants with post mapped metadata were found for this score set. Skipping CAR submission.", + msg="No current alleles found for this score set. Skipping CAR submission.", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"submitted_count": 0, "matched_count": 0}) + return JobExecutionOutcome.succeeded( + data={ + "submitted_allele_count": 0, + "registered_allele_count": 0, + "already_registered_allele_count": 0, + "failed_allele_count": 0, + } + ) - job_manager.update_progress( - 10, 100, f"Preparing {len(variant_post_mapped_objects)} mapped variants for CAR submission." + # Group by allele_id: one allele may serve multiple variants (cross-score-set dedup). + allele_data: dict[int, _AlleleEntry] = {} + for row in allele_rows: + if row.allele_id not in allele_data: + allele_data[row.allele_id] = _AlleleEntry(post_mapped=row.post_mapped, existing_caid=row.clingen_allele_id) + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) - # Build HGVS strings for submission. Don't do duplicate submissions-- store mapped variant IDs by HGVS. - # Variants that can't produce an HGVS string are annotated as failures immediately. - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - variant_post_mapped_hgvs: dict[str, list[int]] = {} - no_hgvs_count = 0 - for mapped_variant_id, post_mapped in variant_post_mapped_objects: - hgvs_for_post_mapped = get_hgvs_from_post_mapped(post_mapped) + # Track outcomes by distinct allele_id. clingen_allele_id is an allele-level fact (the CAID + # lives on the Allele) and CAR's operation is per-allele, so the reported counts are in allele + # units — and they cover every allele submitted, including the RT-derived ones. Each allele has + # exactly one outcome (submitted once → one response), so these sets are disjoint by construction. + linked_allele_ids: set[int] = set() + preexisting_allele_ids: set[int] = set() + failed_allele_ids: set[int] = set() + # Linked alleles whose submitted spelling CAR's response did not list; a subset of linked_allele_ids. + unlisted_allele_ids: set[int] = set() + + preexisting_alleles = [] + pending_alleles = [] + for aid, entry in allele_data.items(): + if entry.existing_caid and not force_reregister: + preexisting_alleles.append(aid) + else: + pending_alleles.append(aid) + + # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. + preexisting = preexisting_alleles if not force_reregister else [] + for allele_id in preexisting: + entry = allele_data[allele_id] + + preexisting_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": entry.existing_caid}, + ) + + job_manager.update_progress(10, 100, f"Preparing {len(pending_alleles)} alleles for CAR submission.") + + # Build HGVS → [allele_ids] map. Multi-variant cis-phased blocks produce no HGVS + # (combine_cis defaults to False); those alleles are annotated as failures immediately. + hgvs_to_allele_ids: dict[str, list[int]] = {} + for allele_id in pending_alleles: + entry = allele_data[allele_id] + hgvs = get_hgvs_from_post_mapped(entry.post_mapped) - if not hgvs_for_post_mapped: - no_hgvs_count += 1 + if hgvs: + hgvs_to_allele_ids.setdefault(hgvs, []).append(allele_id) + + # TODO#780 - Allele is registered but post_mapped can no longer produce HGVS — data + # regression worth surfacing, but the CAID is still valid so treat it as preexisting + # rather than failing the variant. + elif entry.existing_caid: + preexisting_alleles.append(allele_id) logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant_id}. Skipping submission of this variant.", + msg=( + f"Could not construct HGVS for allele {allele_id} during force re-registration " + f"(existing CAID: {entry.existing_caid!r}). Reconfirmation skipped; existing CAID retained." + ), extra=job_manager.logging_context(), ) - - mapped_variant = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id == mapped_variant_id) - ).one() - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Could not extract a valid HGVS string from post-mapped variant data.", - "annotation_metadata": {}, - }, - current=True, + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.RECONFIRMATION_SKIPPED, + metadata={"clingen_allele_id": entry.existing_caid}, ) - continue - if hgvs_for_post_mapped in variant_post_mapped_hgvs: - variant_post_mapped_hgvs[hgvs_for_post_mapped].append(mapped_variant_id) + # No HGVS-- un-submittable. The allele cannot produce an identifier to register, + # so treat this as not_applicable rather than a failure. else: - variant_post_mapped_hgvs[hgvs_for_post_mapped] = [mapped_variant_id] + failed_allele_ids.add(allele_id) + logger.warning( + msg=f"Could not construct HGVS for allele {allele_id}. Skipping CAR submission.", + extra=job_manager.logging_context(), + ) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.NO_HGVS, + error_message="Could not extract a valid HGVS string from post-mapped allele data.", + ) + + job_manager.save_to_context({"unique_hgvs_to_submit_car": len(hgvs_to_allele_ids)}) - job_manager.save_to_context({"unique_variants_to_submit_car": len(variant_post_mapped_hgvs)}) - job_manager.update_progress(15, 100, "Submitting mapped variants to CAR.") + # Distinct alleles actually sent to CAR this run (pending alleles that yielded HGVS). + submitted_allele_ids: set[int] = { + allele_id for allele_ids in hgvs_to_allele_ids.values() for allele_id in allele_ids + } - # Do submission + def _outcome_data() -> dict[str, int]: + return { + "submitted_allele_count": len(submitted_allele_ids), + "registered_allele_count": len(linked_allele_ids), + "already_registered_allele_count": len(preexisting_allele_ids), + "failed_allele_count": len(failed_allele_ids), + "unlisted_hgvs_allele_count": len(unlisted_allele_ids), + } + + # All pending alleles failed HGVS extraction; annotations already written above. + if not hgvs_to_allele_ids: + annotation_manager.flush() + job_manager.db.flush() + if preexisting_allele_ids: + return JobExecutionOutcome.succeeded(data=_outcome_data()) + + return JobExecutionOutcome.failed( + reason=f"No submittable alleles for score set {score_set.urn}.", + data=_outcome_data(), + failure_category=FailureCategory.DEPENDENCY_FAILURE, + ) + + job_manager.update_progress(15, 100, "Submitting alleles to CAR.") car_service = ClinGenAlleleRegistryService(url=CAR_SUBMISSION_ENDPOINT) - hgvs_list = list(variant_post_mapped_hgvs.keys()) - registered_alleles = car_service.dispatch_submissions(hgvs_list) - job_manager.update_progress(60, 100, "Processing registered alleles from CAR.") - - # CAR returns one response per submitted HGVS in the same order (see CAR API docs). - # Zip the submissions with the responses and annotate each based on success or error. - linked_count = 0 - error_count = 0 - - for hgvs_string, response in zip(hgvs_list, registered_alleles): - mapped_variant_ids = variant_post_mapped_hgvs[hgvs_string] - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(mapped_variant_ids)) - ).all() - - if "errorType" in response: - error_count += 1 - logger.warning( - msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + + # Bulk-load every allele that could be linked in one query, keyed by id, rather than + # issuing a SELECT per CAID inside the response loop. + alleles_by_id = { + allele.id: allele + for allele in job_manager.db.scalars(select(AlleleModel).where(AlleleModel.id.in_(submitted_allele_ids))).all() + } + + # Reverse translation can push allele counts high enough that a single PUT is untenable, so + # dispatch in fixed-size chunks. Each batch is reconciled independently: a transient failure + # (or a broken one-result-per-input contract) fails only that batch's alleles, not the run. + batches = list(batched(list(hgvs_to_allele_ids.keys()), DEFAULT_CAR_SUBMISSION_BATCH_SIZE)) + job_manager.save_to_context( + { + "car_submission_batch_size": DEFAULT_CAR_SUBMISSION_BATCH_SIZE, + "car_submission_batch_count": len(batches), + } + ) + + for batch_idx, hgvs_list in enumerate(batches): + # Spread the 15→60 window across batches so progress advances as each chunk is dispatched. + job_manager.update_progress( + 15 + (45 * batch_idx) // len(batches), + 100, + f"Submitting alleles to CAR (batch {batch_idx + 1} / {len(batches)}).", + ) + registered_alleles = car_service.dispatch_submissions(hgvs_list) + + # CAR's contract is one result per submitted HGVS, in order — that is what makes the + # positional zip below valid. A different count (including the empty list dispatch returns + # on request failure) means alignment cannot be trusted for ANY position, so we register + # nothing and fail this batch rather than risk writing a CAID to the wrong allele. + aligned = len(registered_alleles) == len(hgvs_list) + if not aligned: + logger.error( + msg=( + f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " + "positional alignment cannot be trusted. Failing this batch." + ), extra=job_manager.logging_context(), ) - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": { + for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): + allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] + + if "errorType" in response: + logger.warning( + msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + extra=job_manager.logging_context(), + ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.SERVICE_REJECTED, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={ "submitted_hgvs": hgvs_string, "car_error_type": response.get("errorType"), "car_error_message": response.get("message"), }, - }, - current=True, - ) + ) - else: - linked_count += 1 - caid = response["@id"].split("/")[-1] - for mapped_variant in mapped_variants: - mapped_variant.clingen_allele_id = caid - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"clingen_allele_id": caid}}, - current=True, + continue + + # A response that is neither an error nor a registration (no "@id") is malformed. + caid_iri = response.get("@id") + if not caid_iri: + logger.error( + msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", + extra=job_manager.logging_context(), + ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.MALFORMED_RESPONSE, + error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", + metadata={"submitted_hgvs": hgvs_string}, + ) + + continue + + caid = caid_iri.split("/")[-1] + + # CAR lists its own normalized spellings, not the submitted one, so an indel or identity written + # differently from CAR's form is expected to be missing from the list. The positional zip (with the + # count check above) ties the response to the submission; an unlisted spelling is recorded for audit. + spelling_metadata: dict[str, Any] = {} + if hgvs_string not in _response_hgvs(response): + unlisted_allele_ids.update(allele_ids_for_hgvs) + spelling_metadata = {"submitted_hgvs": hgvs_string, "submitted_hgvs_unlisted": True} + logger.debug( + msg=f"CAR returned {caid!r} for HGVS '{hgvs_string}' without listing that spelling.", + extra=job_manager.logging_context(), ) - # Any HGVS strings CAR did not respond to (network drop, service-side omission). - # Use EXTERNAL_SERVICE_REJECTED for explicit CAR errors, EXTERNAL_API_ERROR for silent failures. - no_response_hgvs = hgvs_list[len(registered_alleles) :] - for hgvs_string in no_response_hgvs: - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(variant_post_mapped_hgvs[hgvs_string])) - ).all() - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": {"submitted_hgvs": hgvs_string}, - }, - current=True, - ) + for allele_id in allele_ids_for_hgvs: + entry = allele_data[allele_id] + prior_caid = entry.existing_caid + + # Only force_reregister resubmits an allele that already holds a CAID, and CAR's answer for the + # allele's own HGVS wins. Its gnomAD and ClinVar links were matched through the old CAID, so retire + # them and the later steps link the allele again. + if prior_caid and prior_caid != caid: + linked_allele_ids.add(allele_id) + alleles_by_id[allele_id].clingen_allele_id = caid + retired_gnomad, retired_clinvar = _retire_caid_keyed_links(job_manager.db, allele_id) + logger.warning( + msg=( + f"Replaced CAID {prior_caid!r} with {caid!r} for allele {allele_id}; retired " + f"{retired_gnomad} gnomAD and {retired_clinvar} ClinVar link(s)." + ), + extra=job_manager.logging_context(), + ) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.CORRECTED, + metadata={ + **spelling_metadata, + "clingen_allele_id": caid, + "replaced_caid": prior_caid, + "submitted_hgvs": hgvs_string, + "retired_gnomad_links": retired_gnomad, + "retired_clinvar_links": retired_clinvar, + }, + ) + + # CAID is new or matches the stored value — link it to the allele and record success. + else: + linked_allele_ids.add(allele_id) + allele = alleles_by_id[allele_id] + allele.clingen_allele_id = caid + + reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED + if prior_caid: + logger.info( + msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + extra=job_manager.logging_context(), + ) + + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + reason, + metadata={**spelling_metadata, "clingen_allele_id": caid}, + ) + + # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up + # (network drop, service-side omission), or the entire batch when the response count + # violated CAR's one-result-per-input contract and we rejected it above. + unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] + for hgvs_string in unattributed_hgvs: + for allele_id in hgvs_to_allele_ids[hgvs_string]: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={"submitted_hgvs": hgvs_string}, + ) annotation_manager.flush() - failed_count = no_hgvs_count + error_count + len(no_response_hgvs) + outcome_data = _outcome_data() - # When all registrations fail we will not be able to render any annotations. Fail the job - # to explicitly halt the pipeline. - if linked_count == 0: - error_message = f"CAR submission failed for all {len(hgvs_list)} variants in score set {score_set.urn}." + # When no allele ended up with a CAID (none linked this run, none already registered), the + # pipeline cannot continue — downstream jobs need CAIDs to function. + if not linked_allele_ids and not preexisting_allele_ids: + error_message = ( + f"CAR submission failed for all {outcome_data['submitted_allele_count']} " + f"submitted alleles in score set {score_set.urn}." + ) logger.error(msg=error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": len(hgvs_list), "matched_count": 0, "failed_count": failed_count}, + data=outcome_data, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) - if failed_count > 0: - # CAR rejections are typically per-variant data quality issues (e.g. invalid HGVS) rather than - # systemic failures. Per-variant AnnotationStatus.FAILED records are already written above for - # traceability. We continue the pipeline so that successfully registered variants still receive - # downstream annotations (warm_clingen_cache, gnomAD, ClinVar, HGVS, translations). + if outcome_data["failed_allele_count"] > 0: logger.warning( - msg=f"CAR submission failed for {failed_count} of {len(hgvs_list)} variants in score set {score_set.urn}.", + msg=f"CAR submission failed for {outcome_data['failed_allele_count']} alleles in score set {score_set.urn}.", + extra=job_manager.logging_context(), + ) + + if outcome_data["unlisted_hgvs_allele_count"] > 0: + logger.warning( + msg=( + f"CAR did not list the submitted spelling for {outcome_data['unlisted_hgvs_allele_count']} linked " + f"alleles in score set {score_set.urn}; their events carry submitted_hgvs_unlisted." + ), extra=job_manager.logging_context(), ) logger.info(msg="Completed CAR mapped resource submission", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(hgvs_list), "matched_count": linked_count, "failed_count": failed_count} - ) + return JobExecutionOutcome.succeeded(data=outcome_data) @with_pipeline_management @@ -303,7 +547,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: - Submits data to ClinGen Linked Data Hub Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-variant submitted/failed counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -331,15 +575,26 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_ENDPOINT) ldh_service.authenticate() - # Fetch mapped variants with post-mapped data for the score set - variant_objects = job_manager.db.execute( - select(Variant, MappedVariant) - .join(MappedVariant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) - ).all() + # Fetch each variant's authoritative allele for the score set. Post-mapped data and HGVS + # come from the Allele; pre-mapped data and the mapping API version come from the + # MappingRecord. RT-derived equivalence alleles are intentionally excluded — LDH links each + # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, + # which registers a CAID per allele). + variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( + job_manager.db.execute( + select(Variant, MappingRecord, AlleleModel) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id == score_set.id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(AlleleModel.post_mapped.is_not(None)) + ) + .tuples() + .all() + ) # Track total variants to submit job_manager.save_to_context({"total_variants_to_submit_ldh": len(variant_objects)}) @@ -354,20 +609,22 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.update_progress(10, 100, f"Submitting {len(variant_objects)} mapped variants to LDH.") # Build submission content - variant_content = [] - variant_for_urn = {} - for variant, mapped_variant in variant_objects: - variation = get_hgvs_from_post_mapped(mapped_variant.post_mapped) + variant_content: list[tuple[str, Variant, MappingRecord, AlleleModel]] = [] + variant_for_urn: dict[str, Variant] = {} + for variant, mapping_record, allele in variant_objects: + # cis-phased blocks are skipped here pending ClinGen guidance TODO#764 + variation = get_hgvs_from_post_mapped(allele.post_mapped) if not variation: logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant.id}. Skipping submission of this variant.", + msg=f"Could not construct a valid HGVS string for allele {allele.id} (variant {variant.urn}). Skipping submission of this variant.", extra=job_manager.logging_context(), ) continue - variant_content.append((variation, variant, mapped_variant)) - variant_for_urn[variant.urn] = variant + variant_content.append((variation, variant, mapping_record, allele)) + # TODO#372: nullable URNs + variant_for_urn[variant.urn] = variant # type: ignore if not variant_content: logger.warning( @@ -394,8 +651,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: } ) - # TODO prior to finalizing: Verify typing of ClinGen submission responses. See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + # See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) submitted_variant_urns = set() for success in submission_successes: logger.debug( @@ -404,17 +663,22 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) submitted_urn = success["data"]["entId"] - submitted_variant = variant_for_urn[submitted_urn] + submitted_variant = variant_for_urn.get(submitted_urn) + # LDH echoed back an entId we never submitted — record it for investigation rather + # than crashing the whole job mid-batch. + if submitted_variant is None: + logger.warning( + msg=f"LDH returned an unrecognized entId not in this submission: {submitted_urn!r}.", + extra=job_manager.logging_context(), + ) + continue - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=submitted_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": {"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, - }, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, + metadata={"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, ) submitted_variant_urns.add(submitted_urn) @@ -422,7 +686,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # especially when submission occurred in batch. Save all failures generically here. # Note that failures may not be present in the submission failures list, but they are # guaranteed to be absent from the successes list. - for failure_urn in set(variant_for_urn.keys()) - submitted_variant_urns: + failed_variant_urns = set(variant_for_urn.keys()) - submitted_variant_urns + for failure_urn in failed_variant_urns: logger.error( msg=f"Failed to submit mapped variant to LDH: {failure_urn}", extra=job_manager.logging_context(), @@ -430,23 +695,25 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: failed_variant = variant_for_urn[failure_urn] - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=failed_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to submit variant to ClinGen Linked Data Hub.", - }, - current=True, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "Failed to submit variant to ClinGen Linked Data Hub."}, ) annotation_manager.flush() + # Report per-variant counts (matching the annotations written above), not the per-batch + # counts returned by the service — the two use different denominators. + submitted_count = len(submitted_variant_urns) + failed_count = len(failed_variant_urns) + if submission_failures: logger.warning( - msg=f"LDH mapped resource submission encountered {len(submission_failures)} failures.", + msg=f"LDH mapped resource submission encountered {len(submission_failures)} batch failures " + f"({failed_count} variants unconfirmed).", extra=job_manager.logging_context(), ) @@ -460,7 +727,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": 0, "failed_count": len(submission_failures)}, + data={"submitted_count": submitted_count, "failed_count": failed_count}, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) @@ -470,6 +737,4 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(submission_successes), "failed_count": len(submission_failures)} - ) + return JobExecutionOutcome.succeeded(data={"submitted_count": submitted_count, "failed_count": failed_count}) diff --git a/src/mavedb/worker/jobs/external_services/clingen_cache.py b/src/mavedb/worker/jobs/external_services/clingen_cache.py index 10890f23f..11545ec05 100644 --- a/src/mavedb/worker/jobs/external_services/clingen_cache.py +++ b/src/mavedb/worker/jobs/external_services/clingen_cache.py @@ -14,12 +14,11 @@ from sqlalchemy import select -from mavedb.lib.clingen.allele_registry import get_clingen_allele_data +from mavedb.lib.clingen.allele_registry import get_clingen_allele_data, refresh_clingen_allele_data_expiry +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import CLINGEN_CACHE_WARMING_CONCURRENCY from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -55,19 +54,16 @@ async def warm_clingen_cache(ctx: dict, job_id: int, job_manager: JobManager) -> job_manager.update_progress(0, 100, "Starting ClinGen cache pre-warming.") logger.info("Starting ClinGen cache pre-warming", extra=job_manager.logging_context()) - # Get distinct clingen_allele_ids for this score set's current mapped variants - allele_ids = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.isnot(None), - # Exclude multi-variant IDs (comma-separated) — they can't be fetched individually - MappedVariant.clingen_allele_id.not_like("%,%"), - ) - .distinct() - ).all() + # Get distinct clingen_allele_ids registered for alleles in this score set. + # Exclude None and comma-separated multi-variant IDs that can't be fetched individually. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_ids = list( + { + row.clingen_allele_id + for row in allele_rows + if row.clingen_allele_id is not None and "," not in row.clingen_allele_id + } + ) total = len(allele_ids) job_manager.save_to_context({"total_allele_ids_to_warm": total}) @@ -78,13 +74,15 @@ async def warm_clingen_cache(ctx: dict, job_id: int, job_manager: JobManager) -> return JobExecutionOutcome.succeeded(data={"warmed": 0, "failed": 0}) # Fetch alleles concurrently up to CLINGEN_CACHE_WARMING_CONCURRENCY in-flight at a time. - # get_clingen_allele_data() is decorated with @cached, so each call populates Redis. + # get_clingen_allele_data() is decorated with @cached, so each call populates Redis. A hit keeps + # the expiry its first writer set, so restart it: the downstream jobs need it warm, not just present. semaphore = asyncio.Semaphore(CLINGEN_CACHE_WARMING_CONCURRENCY) async def fetch_one(allele_id: str) -> tuple[str, bool, BaseException | None]: async with semaphore: try: await get_clingen_allele_data(allele_id) + await refresh_clingen_allele_data_expiry(allele_id) return allele_id, True, None except Exception as exc: return allele_id, False, exc diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 68a34a04b..5aadfd8ca 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -1,19 +1,18 @@ -"""ClinVar integration jobs for variant annotation +"""ClinVar integration jobs for variant annotation. -This module contains job definitions and utility functions for integrating ClinVar -variant data into MaveDB. It includes functions to fetch and parse ClinVar variant -summary data, and update MaveDB records with the latest ClinVar annotations. +Links deduplicated alleles to ClinVar clinical-control data across every archival ClinVar +release. Each release is a distinct, versioned ``ClinvarControl`` entity, and an allele +accumulates one live ``ClinvarAlleleLink`` per release it appears in (multi-live, unlike +gnomAD/VEP which hold one live result per allele). Both ClinGen API calls and ClinVar TSV data fetches are automatically cached using aiocache with Redis backend: - ClinGen API calls: 24-hour TTL - ClinVar TSV files: 90-day TTL (archival data doesn't change) - -This significantly reduces redundant network requests when refreshing ClinVar -controls across multiple months/years. """ import logging +from collections import Counter from datetime import datetime import requests @@ -22,48 +21,97 @@ from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.clingen.allele_registry import get_associated_clinvar_allele_id +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager logger = logging.getLogger(__name__) +# Alleles between progress checkpoints within a version. The first version of a run can fetch +# thousands of uncached ClinGen lookups; without checkpoints the stall reaper cancels it at 30 min. +CLINVAR_PROGRESS_EVERY_ALLELES = 500 # ClinVar archived data starts from February 2015, then January of each # subsequent year. This list is used to generate the date range for refreshing. CLINVAR_START_YEAR = 2015 CLINVAR_START_MONTH = 2 -def generate_clinvar_versions() -> list[tuple[int, int]]: +def _generate_clinvar_versions() -> list[tuple[int, int]]: """Generate all ClinVar version (year, month) pairs from Feb 2015 to current Jan. Returns a list of (year, month) tuples representing each ClinVar archival snapshot that should be processed. """ current_year = datetime.now().year - versions = [(CLINVAR_START_YEAR, CLINVAR_START_MONTH)] - for year in range(CLINVAR_START_YEAR + 1, current_year + 1): - versions.append((year, 1)) - return versions + first_version = (CLINVAR_START_YEAR, CLINVAR_START_MONTH) + archival_versions = [(year, 1) for year in range(CLINVAR_START_YEAR + 1, current_year + 1)] + return [first_version, *archival_versions] + + +def _annotate_clinvar( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + source_version: str, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CLINVAR_CONTROL event for an allele at a given ClinVar release. + + The single choke point for ClinVar's status writes. One event per (allele, release); provenance + (which variants drove the linkage) is derived from the live links as-of the event, not fanned out + here. Unlike gnomAD/VEP, ClinVar is multi-version, so ``source_version`` (the ClinVar release) + distinguishes an allele's events across releases. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINVAR_CONTROL, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) @with_pipeline_management async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Refresh ClinVar clinical control data across all archival versions. - - Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each - subsequent year through the current year), fetching TSV data and updating - clinical control records for all mapped variants in the score set. Individual - version failures are logged and skipped — the job continues processing - remaining versions. + """Link deduplicated alleles to ClinVar clinical-control data across all archival versions. + + Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each subsequent year through + the current year). For each version it resolves each allele's ClinGen Allele ID (CAID) to a ClinVar + allele id, upserts the versioned :class:`ClinvarControl`, and establishes a live + :class:`ClinvarAlleleLink`. An allele accumulates one live link per release (multi-live). Individual + version fetch failures are logged and skipped — the job continues with the remaining versions. + + Job Parameters: + - score_set_id (int): The ID of the ScoreSet whose alleles to process. + - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the per-version skip and re-resolve every allele. The link + write still get-or-creates (no duplicate live link), so a forced re-run of unchanged data + writes no new links. + + Side Effects: + - Creates ClinvarControl and ClinvarAlleleLink rows. + + Returns: + JobExecutionOutcome: outcome with version and per-link counts. """ job = job_manager.get_job() @@ -72,8 +120,9 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] - versions = generate_clinvar_versions() + versions = _generate_clinvar_versions() job_manager.save_to_context( { @@ -83,26 +132,56 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag "correlation_id": correlation_id, "versions": versions, "total_versions": len(versions), + "force": force, } ) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) - variants_to_refresh = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), + # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys + # on is often the RT-derived one. Events are allele-keyed, so every allele is recorded per release + # (no fan-out). + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} + job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) + + # Link counts accumulate across all versions (an allele may link in every release it appears in). + annotation_counts: Counter[str] = Counter( + { + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + } + ) + + if not allele_data: + logger.warning( + msg="No current alleles with CAIDs were found for this score set. Skipping ClinVar refresh.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded( + data={"versions_completed": 0, "versions_total": len(versions), **dict(annotation_counts)} + ) + + all_allele_ids = set(allele_data.keys()) + + def alleles_linked_at_version(clinvar_version: str) -> set[int]: + """Allele ids (within the work set) holding a live ClinvarAlleleLink to a control of this version.""" + return set( + job_manager.db.scalars( + select(ClinvarAlleleLink.allele_id) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(all_allele_ids)) + .where(ClinvarAlleleLink.current) + .where(ClinvarControl.db_version == clinvar_version) + ).all() ) - ).all() - total_variants_to_refresh = len(variants_to_refresh) - job_manager.save_to_context({"total_variants_to_refresh": total_variants_to_refresh}) - total_refreshed = 0 - total_failed = 0 versions_completed = 0 - for version_index, (year, month) in enumerate(versions): clinvar_version = f"{month:02d}_{year}" job_manager.save_to_context({"current_version": clinvar_version, "version_index": version_index}) @@ -125,163 +204,212 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag ) continue - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in variants_to_refresh: - clingen_id = mapped_variant.clingen_allele_id - - if clingen_id is None: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Mapped variant does not have an associated ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + # Cost: skip alleles already linked at this version (an archival release cannot change). force + # bypasses the skip but the link write still get-or-creates, so a forced no-op writes nothing. + already_linked = set() if force else alleles_linked_at_version(clinvar_version) + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_index, (allele_id, caid) in enumerate(allele_data.items(), start=1): + if allele_index % CLINVAR_PROGRESS_EVERY_ALLELES == 0: + job_manager.update_progress( + int((version_index + allele_index / len(allele_data)) / len(versions) * 100), + 100, + f"Processing ClinVar version {clinvar_version} ({version_index + 1}/{len(versions)}): " + f"{allele_index}/{len(allele_data)} alleles.", + ) + + # ClinVar is a nucleotide-level database, skip any protein level alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + source_version=clinvar_version, + error_message="Protein-level alleles are not linked to ClinVar; see the nucleotide encodings.", + metadata={"clingen_allele_id": caid}, ) continue - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", - }, - current=True, - replace_all_versions=False, + if allele_id in already_linked: + annotation_counts["preexisting_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid}, + ) + continue + + # A cis-block (multi-variant) CAID structurally cannot key ClinVar — a terminal gap, not + # a statement about ClinVar's contents. + if "," in caid: + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.MULTI_VARIANT_CAID, + source_version=clinvar_version, + error_message="Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", + metadata={"clingen_allele_id": caid}, ) continue try: - clinvar_allele_id = await get_associated_clinvar_allele_id(clingen_id) # type: ignore + clinvar_allele_id = await get_associated_clinvar_allele_id(caid) except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", - }, - current=True, - replace_all_versions=False, + annotation_counts["failed_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=clinvar_version, + error_message=f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", + metadata={"clingen_allele_id": caid}, ) logger.error( - f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {clingen_id}.", + f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {caid}.", extra=job_manager.logging_context(), exc_info=exc, ) - total_failed += 1 continue + # ClinGen has no ClinVar AlleleID for this CAID — the allele is not a ClinVar control: an + # informative negative about the source, not a pipeline gap. if not clinvar_allele_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": "No ClinVar allele ID found for ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, + error_message="No ClinVar allele ID found for ClinGen allele ID.", + metadata={"clingen_allele_id": caid}, ) continue + # The allele has a ClinVar AlleleID but it is absent from this release's snapshot — a + # genuine, version-scoped negative (ClinVar queried, nothing for this release). if clinvar_allele_id not in tsv_data: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No ClinVar data found for ClinVar allele ID.", - }, - current=True, - replace_all_versions=False, + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, + error_message="No ClinVar data found for ClinVar allele ID.", + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, ) continue variant_data = tsv_data[clinvar_allele_id] - identifier = str(clinvar_allele_id) - # Atomic upsert — avoids a check-then-act race when two - # refresh_clinvar_controls jobs run concurrently for different - # score sets and encounter the same (db_name, db_identifier, - # db_version) tuple. ON CONFLICT DO UPDATE is guaranteed to - # return exactly one row regardless of concurrent inserts. + # Atomic upsert — avoids a check-then-act race when two refresh_clinvar_controls jobs run + # concurrently for different score sets and encounter the same + # (db_name, db_identifier, db_version) tuple. ON CONFLICT DO UPDATE returns exactly one row. upsert_stmt = ( - pg_insert(ClinicalControl) + pg_insert(ClinvarControl) .values( - db_identifier=identifier, + db_identifier=str(clinvar_allele_id), db_version=clinvar_version, db_name="ClinVar", gene_symbol=variant_data.get("GeneSymbol"), clinical_significance=variant_data.get("ClinicalSignificance"), clinical_review_status=variant_data.get("ReviewStatus"), + clinvar_variation_id=variant_data.get("VariationID"), ) .on_conflict_do_update( - constraint="uq_clinical_controls_db_name_identifier_version", + constraint="uq_clinvar_controls_db_name_identifier_version", set_={ "gene_symbol": variant_data.get("GeneSymbol"), "clinical_significance": variant_data.get("ClinicalSignificance"), "clinical_review_status": variant_data.get("ReviewStatus"), + "clinvar_variation_id": variant_data.get("VariationID"), }, ) - .returning(ClinicalControl) + .returning(ClinvarControl) ) - clinvar_variant = job_manager.db.scalars(upsert_stmt).one() - - job_manager.db.add(clinvar_variant) - job_manager.db.flush() - - if clinvar_variant not in mapped_variant.clinical_controls: - mapped_variant.clinical_controls.append(clinvar_variant) - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "clinvar_allele_id": clinvar_allele_id, - }, - }, - current=True, - replace_all_versions=False, + clinvar_control = job_manager.db.scalars(upsert_stmt).one() + + # At most one live link per (allele, release). Normally a release is immutable, so the + # allele's live link for this version is either a reconfirm of this same control (no-op) or + # absent (insert) — multi-live accumulates only across *different* releases, never supersedes. + # Defensive guard: if the allele already holds a live link to a *different* control of this + # same version, the release re-resolved under us (re-ingestion / upstream correction — should + # never happen for archival data). Supersede newest-wins with a shared timestamp and log. + live_link = job_manager.db.scalar( + select(ClinvarAlleleLink) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.current, + ClinvarControl.db_version == clinvar_version, + ) ) + if live_link is None: + job_manager.db.add(ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id)) + annotation_counts["created_link_count"] += 1 + reason = EventReason.CREATED + elif live_link.clinvar_control_id == clinvar_control.id: + annotation_counts["preexisting_link_count"] += 1 + reason = EventReason.PREEXISTING + else: + live_link.supersede_with( + job_manager.db, + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id), + ) + annotation_counts["created_link_count"] += 1 + reason = EventReason.SUPERSEDED + logger.warning( + msg=( + f"Allele {allele_id} held a live ClinVar link to control " + f"{live_link.clinvar_control_id} for version {clinvar_version}, but re-resolved to " + f"control {clinvar_control.id} (ClinVar allele {clinvar_allele_id}). Superseding " + "newest-wins; archival ClinVar data should be immutable — investigate the upstream " + "re-resolution." + ), + extra=job_manager.logging_context(), + ) - total_refreshed += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.PRESENT, + reason, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, + ) annotation_manager.flush() + job_manager.db.flush() versions_completed += 1 logger.info( - f"Completed ClinVar version {clinvar_version} for {total_variants_to_refresh} variants.", + f"Completed ClinVar version {clinvar_version} for {len(allele_data)} alleles.", extra=job_manager.logging_context(), ) logger.info( f"ClinVar refresh complete: {versions_completed}/{len(versions)} versions, " - f"{total_refreshed} variant-version annotations.", + f"{annotation_counts['created_link_count']} new links, " + f"{annotation_counts['preexisting_link_count']} preexisting.", extra=job_manager.logging_context(), ) - if total_failed > 0 and total_refreshed == 0: - error_message = ( - f"All {total_failed} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - ) + if ( + annotation_counts["failed_link_count"] > 0 + and annotation_counts["created_link_count"] == 0 + and annotation_counts["preexisting_link_count"] == 0 + ): + error_message = f"All {annotation_counts['failed_link_count']} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." logger.error(error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -289,16 +417,12 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag data={ "versions_completed": versions_completed, "versions_total": len(versions), - "variant_annotations": 0, + **dict(annotation_counts), }, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "versions_completed": versions_completed, - "versions_total": len(versions), - "variant_annotations": total_refreshed, - } + data={"versions_completed": versions_completed, "versions_total": len(versions), **dict(annotation_counts)} ) diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 969c3a20e..4c688692d 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -1,29 +1,33 @@ """gnomAD variant linking jobs for population frequency annotation. -This module handles linking of mapped variants to gnomAD (Genome Aggregation Database) +This module handles linking of deduplicated alleles to gnomAD (Genome Aggregation Database) variants to provide population frequency and other genomic context information. This enrichment helps researchers understand the clinical significance and rarity of variants in their datasets. """ import logging -from typing import Sequence +from collections import Counter from sqlalchemy import select from mavedb.db import athena from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.gnomad import ( GNOMAD_DATA_VERSION, + GnomadLinkVerdict, gnomad_variant_data_for_caids, - link_gnomad_variants_to_mapped_variants, + link_gnomad_variants_to_alleles, ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -31,17 +35,48 @@ logger = logging.getLogger(__name__) +def _annotate_gnomad( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one GNOMAD_ALLELE_FREQUENCY event for an allele (frequency is an allele-level fact). + + The single choke point for gnomAD's status writes. One event per allele, stamped at the current + gnomAD version; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=GNOMAD_DATA_VERSION, + metadata=meta or None, + ) + + @with_pipeline_management async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ - Link mapped variants to gnomAD variants based on ClinGen Allele IDs (CAIDs). - This job fetches mapped variants associated with a given score set that have CAIDs, - retrieves corresponding gnomAD variant data, and establishes links between them - in the database. + Link deduplicated alleles to gnomAD variants based on ClinGen Allele IDs (CAIDs). + This job fetches the current authoritative alleles of a score set that carry CAIDs, + retrieves corresponding gnomAD variant data, and establishes valid-time links between them. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants to process. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the version-keyed skip and re-fetch every CAID-bearing + allele. The linker still supersedes only on change, so a forced re-run of unchanged data + writes no new links. Use for re-ingestion or to heal suspected link corruption. Args: ctx (dict): The job context dictionary. @@ -49,10 +84,10 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager (JobManager): The job manager instance for database and logging operations. Side Effects: - - Updates MappedVariant records to link to gnomAD variants. + - Creates GnomadAlleleLink rows linking alleles to gnomAD variants. Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -63,6 +98,7 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] # Setup initial context and progress job_manager.save_to_context( @@ -76,87 +112,144 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager.update_progress(0, 100, "Starting gnomAD mapped resource linkage.") logger.info(msg="Started gnomAD mapped resource linkage", extra=job_manager.logging_context()) - # We filter out mapped variants that do not have a CAID, so this query is typed # as a Sequence[str]. Ignore MyPy's type checking here. - variant_caids: Sequence[str] = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), - ) - ).all() # type: ignore + # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows + # is often the RT-derived one. Events are allele-keyed, so every linked allele is recorded. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} + + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "skipped_allele_count": 0, + } + ) - num_variant_caids = len(variant_caids) - job_manager.save_to_context({"num_variants_to_link_gnomad": num_variant_caids}) + num_alleles_with_caids = len(allele_data) + job_manager.save_to_context({"num_alleles_to_link_gnomad": num_alleles_with_caids}) - if not variant_caids: + if not allele_data: logger.warning( - msg="No current mapped variants with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", + msg="No current alleles with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"linked_count": 0, "skipped_count": 0}) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) - job_manager.update_progress(10, 100, f"Found {num_variant_caids} variants with CAIDs to link to gnomAD variants.") - logger.info( - msg="Found current mapped variants with CAIDs for this score set. Attempting to link them to gnomAD variants.", - extra=job_manager.logging_context(), + job_manager.update_progress( + 10, 100, f"Found {num_alleles_with_caids} alleles with CAIDs to link to gnomAD variants." ) - # Fetch gnomAD variant data for the CAIDs - with athena.engine.connect() as athena_session: - logger.debug("Fetching gnomAD variants from Athena.") - gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) + def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live gnomAD link at the current gnomAD version.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(GnomadAlleleLink.allele_id) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.current) + .where(GnomADVariant.db_version == GNOMAD_DATA_VERSION) + ).all() + ) - num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + # Skip alleles already linked at the current version (they can't change). force re-fetches + # all; the linker still supersedes only on change, so a forced no-op writes nothing. + already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) + # Never query Athena for protein alleles — gnomAD is nucleotide-level, so they can't match. They + # are still recorded as not-applicable in the loop below; this only spares them the round-trip. + variant_caids = sorted( + { + allele_data[aid] + for aid in allele_data + if aid not in already_current and allele_levels.get(aid) != SequenceLevel.protein.value + } + ) - # NOTE: Proceed intentionally with linking even if no matches were found, to record skipped annotations. + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_caids_to_query": len(variant_caids), + "force": force, + } + ) - job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) - job_manager.update_progress(75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs.") + verdicts: dict[int, GnomadLinkVerdict] = {} + if variant_caids: + with athena.engine.connect() as athena_session: + logger.debug("Fetching gnomAD variants from Athena.") + gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) - # Link mapped variants to gnomAD variants - logger.info(msg="Attempting to link mapped variants to gnomAD variants.", extra=job_manager.logging_context()) - num_linked_gnomad_variants = link_gnomad_variants_to_mapped_variants(job_manager.db, gnomad_variant_data) - job_manager.db.flush() + num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) + job_manager.update_progress( + 75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs." + ) - # For variants which are not linked, create annotation status records indicating skipped linkage - mapped_variants_with_caids = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), + logger.info(msg="Attempting to link alleles to gnomAD variants.", extra=job_manager.logging_context()) + verdicts = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) + job_manager.db.flush() + else: + logger.info( + msg="All CAID-bearing alleles are already linked at the current gnomAD version; skipping Athena query.", + extra=job_manager.logging_context(), ) - ).all() - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in mapped_variants_with_caids: - if not mapped_variant.gnomad_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No gnomAD variant could be linked for this mapped variant.", - }, - current=True, + job_manager.update_progress(75, 100, "All alleles already current at this gnomAD version.") + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, caid in allele_data.items(): + # gnomAD is a nucleotide-level, genomic-coordinate resource. Skip any protein alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + error_message="Protein-level alleles are not linked to gnomAD; see the nucleotide encodings.", + metadata={"clingen_allele_id": caid}, + ) + continue + + verdict = verdicts.get(allele_id) + if verdict is GnomadLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + metadata={"clingen_allele_id": caid}, + ) + elif allele_id in already_current or verdict is GnomadLinkVerdict.UNCHANGED: + annotation_counts["preexisting_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": caid}, + ) + else: + annotation_counts["skipped_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + error_message="No gnomAD variant could be linked for this allele.", + metadata={"clingen_allele_id": caid}, ) annotation_manager.flush() - # Save final context and progress - job_manager.save_to_context({"num_mapped_variants_linked_to_gnomad_variants": num_linked_gnomad_variants}) - logger.info(msg="Done linking gnomAD variants to mapped variants.", extra=job_manager.logging_context()) + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) + logger.info(msg="Done linking gnomAD variants to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "linked_count": num_linked_gnomad_variants, - "skipped_count": num_variant_caids - num_linked_gnomad_variants, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/src/mavedb/worker/jobs/external_services/hgvs.py b/src/mavedb/worker/jobs/external_services/hgvs.py deleted file mode 100644 index 0b4687398..000000000 --- a/src/mavedb/worker/jobs/external_services/hgvs.py +++ /dev/null @@ -1,298 +0,0 @@ -"""ClinGen allele HGVS population jobs for mapped variant annotation. - -This module populates mapped variants with HGVS representations (genomic, coding, -protein) by querying the ClinGen Allele Registry. It uses ClinGen allele IDs -(CAIDs) already associated with mapped variants to look up standardized HGVS -nomenclature at different levels (hgvs_g, hgvs_c, hgvs_p), plus the assay-level -HGVS derived from post-mapped VRS data. -""" - -import logging -from typing import Optional - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - extract_hgvs_from_ca_allele_data, - extract_hgvs_from_pa_allele_data, - get_clingen_allele_data, -) -from mavedb.lib.slack import log_and_send_slack_message -from mavedb.lib.target_genes import get_target_coding_info -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_hgvs_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate mapped variants with HGVS representations for a score set. - - Queries the ClinGen Allele Registry using existing ClinGen allele IDs to populate - standardized HGVS nomenclature (genomic, coding, protein) on mapped variants. - Also extracts the assay-level HGVS from post-mapped VRS data. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - - Args: - ctx: Worker context containing DB and Redis connections. - job_id: The ID of the job run. - job_manager: Manager for job lifecycle and DB operations. - - Side Effects: - - Updates MappedVariant records with hgvs_assay_level, hgvs_g, hgvs_c, hgvs_p. - - Creates AnnotationStatus records for each processed variant. - - Returns: - JobExecutionOutcome indicating success, failure, or skip. - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - # Setup initial context and progress - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_hgvs_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting mapped HGVS population.") - logger.info(msg="Started mapped HGVS population", extra=job_manager.logging_context()) - - # Determine target info; multi-target score sets are not yet supported - try: - target_is_coding, transcript_accession = get_target_coding_info(score_set) - except NotImplementedError: - logger.warning( - msg="Multi-target score sets not supported for HGVS population. Skipping.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.skipped(data={"reason": "Multi-target score sets not supported"}) - - job_manager.save_to_context({"target_is_coding": target_is_coding, "transcript_accession": transcript_accession}) - logger.info( - msg=f"Target info resolved: coding={target_is_coding}, transcript={transcript_accession}", - extra=job_manager.logging_context(), - ) - - # Fetch current mapped variants for the score set - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - total_variants = len(variant_rows) - job_manager.save_to_context({"total_variants": total_variants}) - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set. Skipping HGVS population.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"populated_count": 0, "skipped_count": 0, "failed_count": 0}) - - job_manager.update_progress(5, 100, f"Processing {total_variants} mapped variants for HGVS population.") - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - populated_count = 0 - skipped_count = 0 - failed_count = 0 - - for index, (variant_id, mapped_variant) in enumerate(variant_rows): - # Periodic progress updates - if total_variants > 0 and index % max(total_variants // 20, 1) == 0: - progress = 5 + int((index / total_variants) * 90) - job_manager.update_progress(progress, 100, f"Processing HGVS for variant {index + 1}/{total_variants}.") - logger.info( - "Processing variant %s/%s: variant_id=%s", - index + 1, - total_variants, - variant_id, - extra=job_manager.logging_context(), - ) - - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - clingen_id = mapped_variant.clingen_allele_id - - job_manager.save_to_context( - { - "mapped_variant_id": mapped_variant.id, - "clingen_allele_id": clingen_id, - "progress_index": index, - } - ) - - if not clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "No ClinGen allele ID available for ClinGen HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: no ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Skip multi-variant allele IDs (comma-separated) - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs not supported for HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: multi-variant ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Query ClinGen API for allele data - try: - allele_data = await get_clingen_allele_data(clingen_id) - except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to fetch ClinGen allele data: {str(exc)}", - }, - current=True, - ) - logger.error( - "ClinGen API request failed for allele %s.", - clingen_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed_count += 1 - continue - - if allele_data is None: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": f"ClinGen allele {clingen_id} not found in the registry.", - }, - current=True, - ) - logger.debug( - "ClinGen allele %s not found in registry. Skipping variant %s.", - clingen_id, - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Extract HGVS based on allele type - if clingen_id.startswith("CA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_ca_allele_data( - allele_data, target_is_coding, transcript_accession - ) - elif clingen_id.startswith("PA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_pa_allele_data(allele_data) - - # Update mapped variant - mapped_variant.hgvs_g = hgvs_g - mapped_variant.hgvs_c = hgvs_c - mapped_variant.hgvs_p = hgvs_p - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "hgvs_g": hgvs_g, - "hgvs_c": hgvs_c, - "hgvs_p": hgvs_p, - }, - }, - current=True, - ) - populated_count += 1 - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) - logger.info( - msg=f"Completed mapped HGVS population: {populated_count} populated, {skipped_count} skipped, {failed_count} failed.", - extra=job_manager.logging_context(), - ) - - if failed_count > 0 and populated_count == 0: - log_and_send_slack_message( - f"All {failed_count} variants failed HGVS population for score set {score_set.urn}. Possible ClinGen API outage.", - job_manager.logging_context(), - logging.ERROR, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) diff --git a/src/mavedb/worker/jobs/external_services/uniprot.py b/src/mavedb/worker/jobs/external_services/uniprot.py index e16984292..5e7a5ab2a 100644 --- a/src/mavedb/worker/jobs/external_services/uniprot.py +++ b/src/mavedb/worker/jobs/external_services/uniprot.py @@ -17,7 +17,7 @@ from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.uniprot.id_mapping import UniProtIDMappingAPI -from mavedb.lib.uniprot.utils import infer_db_name_from_sequence_accession +from mavedb.lib.uniprot.utils import id_mapping_query_accession, infer_db_name_from_sequence_accession from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.job_dependency import JobDependency from mavedb.models.score_set import ScoreSet @@ -123,8 +123,9 @@ async def submit_uniprot_mapping_jobs_for_score_set( ) continue - ac_to_map = acs[0] - from_db = infer_db_name_from_sequence_accession(ac_to_map) + from_db = infer_db_name_from_sequence_accession(acs[0]) + # Store the submitted form: polling matches results by the accession UniProt echoes back. + ac_to_map = id_mapping_query_accession(acs[0], from_db) spawned_job = uniprot_api.submit_id_mapping(from_db, "UniProtKB", [ac_to_map]) # type: ignore # Explicitly cast ints to strs in mapping job keys. These are converted to strings internally diff --git a/src/mavedb/worker/jobs/external_services/variant_translation.py b/src/mavedb/worker/jobs/external_services/variant_translation.py deleted file mode 100644 index ddec4b731..000000000 --- a/src/mavedb/worker/jobs/external_services/variant_translation.py +++ /dev/null @@ -1,371 +0,0 @@ -"""ClinGen allele variant translation jobs for mapping PA<->CA allele relationships. - -This module populates the variant_translations table with relationships between -protein allele (PA) and nucleotide allele (CA) ClinGen IDs. For CA alleles, it -looks up MANE canonical PA IDs and their matching registered transcript CA IDs. -For PA alleles, it looks up matching registered transcript CA IDs directly. -""" - -import logging - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - expand_allele_ids, - get_canonical_pa_ids, - get_matching_registered_ca_ids, -) -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.lib.variant_translations import upsert_variant_translations -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_variant_translations_for_score_set( - ctx: dict, job_id: int, job_manager: JobManager -) -> JobExecutionOutcome: - """Populate variant translations (PA<->CA relationships) for a score set. - - Queries the ClinGen Allele Registry to discover relationships between protein - allele (PA) and nucleotide allele (CA) ClinGen IDs, then stores them in the - variant_translations table. Each unique allele ID is processed once even if - shared across multiple mapped variants. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_variant_translations_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting variant translation population.") - logger.info(msg="Started variant translation population.", extra=job_manager.logging_context()) - - # Fetch all current mapped variants with their ClinGen allele IDs - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant.clingen_allele_id) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .join(ScoreSet, Variant.score_set_id == ScoreSet.id) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - # Deduplicate: multiple mapped variants can share the same allele ID, but we only - # need to query the ClinGen API once per unique ID. Track which variants map to each - # allele so we can record annotations for all of them after a single lookup. - allele_to_variants: dict[str, list[int]] = {} - for variant_id, clingen_allele_id in variant_rows: - if not clingen_allele_id: - continue - - for individual_id in expand_allele_ids([clingen_allele_id]): - allele_to_variants.setdefault(individual_id, []).append(variant_id) - - unique_allele_ids = list(allele_to_variants.keys()) - total_alleles = len(unique_allele_ids) - job_manager.save_to_context({"total_variants": len(variant_rows), "unique_allele_ids": total_alleles}) - - if not unique_allele_ids: - logger.warning( - msg="No ClinGen allele IDs found on mapped variants.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - job_manager.update_progress(5, 100, f"Processing {total_alleles} unique allele IDs for variant translations.") - logger.info( - "Processing %s unique allele IDs for variant translations.", - total_alleles, - extra=job_manager.logging_context(), - ) - - total_created = 0 - total_skipped = 0 - total_failed = 0 - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - for index, allele_id in enumerate(unique_allele_ids): - if total_alleles > 0 and index % max(total_alleles // 20, 1) == 0: - progress = 5 + int((index / total_alleles) * 90) - job_manager.update_progress(progress, 100, f"Processing allele {index + 1}/{total_alleles}.") - logger.info( - "Processing allele %s/%s: %s", - index + 1, - total_alleles, - allele_id, - extra=job_manager.logging_context(), - ) - - job_manager.save_to_context( - { - "current_allele_id": allele_id, - "progress_index": index, - } - ) - - variant_ids = allele_to_variants[allele_id] - - if allele_id.startswith("CA"): - # CA (nucleotide) alleles: look up the MANE canonical protein alleles (PAs) for - # this CA, then for each PA discover all registered transcript-level CAs. This - # CA -> PA -> CA expansion builds the full translation graph so we can link - # nucleotide variants to their protein equivalents and vice versa. - try: - canonical_pa_ids = await get_canonical_pa_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for canonical PA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error looking up PA IDs for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not canonical_pa_ids: - # Noncoding variants won't have protein alleles — this is expected and not an error. - logger.debug( - "No canonical PA IDs found for %s (may be noncoding).", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No canonical PA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - created = 0 - failed = 0 - translation_pairs: set[tuple[str, str]] = set() - for pa_id in canonical_pa_ids: - # Record the direct PA <-> original CA relationship. - translation_pairs.add((pa_id, allele_id)) - - # Then expand: find all other CAs registered under this PA so we capture - # alternate transcript-level representations of the same protein change. - try: - ca_ids = await get_matching_registered_ca_ids(pa_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - pa_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed += 1 - continue - - for ca_id in ca_ids: - translation_pairs.add((pa_id, ca_id)) - - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED if failed > 0 else AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - "pa_lookups_failed": failed, - "pa_lookups_total": len(canonical_pa_ids), - }, - }, - current=True, - ) - - total_created += created - total_failed += failed - - elif allele_id.startswith("PA"): - # PA (protein) alleles: directly look up all registered transcript-level CAs. - # This is simpler than the CA path since we already have the protein allele. - try: - ca_ids = await get_matching_registered_ca_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not ca_ids: - logger.warning( - "No matching registered transcript CA IDs for PA allele %s. This is unexpected.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No registered transcript CA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - translation_pairs = set([(allele_id, ca_id) for ca_id in ca_ids]) - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - }, - }, - current=True, - ) - - total_created += created - - else: - logger.warning( - "Unrecognized ClinGen allele ID format: %s. Skipping.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": f"Unrecognized allele ID format: {allele_id}", - }, - current=True, - ) - total_skipped += len(variant_ids) - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) - logger.info( - "Completed variant translation population: %s created, %s skipped, %s failed.", - total_created, - total_skipped, - total_failed, - extra=job_manager.logging_context(), - ) - - if total_failed > 0 and total_created == 0: - error_message = f"All {total_failed} variant translation lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - logger.error(error_message, extra=job_manager.logging_context()) - job_manager.db.flush() - return JobExecutionOutcome.failed( - reason=error_message, - data={ - "translations_created": 0, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - }, - failure_category=FailureCategory.DEPENDENCY_FAILURE, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index ba577f230..a6195d1b6 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -1,50 +1,188 @@ -"""VEP functional consequence jobs for variant effect prediction. +"""VEP molecular-consequence job for variant effect prediction. -This module handles the submission and processing of variant effect predictions -using the Ensembl VEP API. - -The processing is asynchronous, requiring batch submission of HGVS strings -to the VEP API with fallback to Variant Recoder when necessary. +Links deduplicated alleles to their Ensembl VEP molecular consequence. The job owns the lifecycle — +selecting which alleles to annotate, the current-release skip, linking, and status events — while the +resolution itself (transport, batching, Recoder fallback, the transcript-matching rule) lives in +``mavedb.lib.vep`` and the shared ``variant_annotation.lib.vep`` kernel, so the pipeline and the lab +CLI produce identical consequences for the same input (#772). """ -import asyncio +import contextlib import logging -import os +from collections import Counter from datetime import date +from typing import Iterator, Optional from sqlalchemy import select +from variant_annotation.lib.vep import ( + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ReferenceSequence, + VepInput, +) from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.utils import batched -from mavedb.lib.vep import VEP_CONSEQUENCES, get_functional_consequence, run_variant_recoder +from mavedb.lib.vep import ( + VepLinkVerdict, + get_ensembl_release, + link_vep_consequences_to_alleles, + resolve_consequences, +) from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.translation_ports import uta_transcript_source logger = logging.getLogger(__name__) -_VEP_BATCH_SIZE = 200 -_RECODER_BATCH_SIZE = int(os.getenv("RECODER_BATCH_SIZE", "25")) -_RECODER_CONCURRENCY = int(os.getenv("RECODER_CONCURRENCY", "5")) +# Inputs resolved per await so progress can be reported between chunks. Matches the kernel's own VEP +# batch ceiling (200 HGVS/request), so each chunk is ~one wire batch. Progress must be reported from +# this coroutine, never from inside resolve_consequences: that call runs on an executor thread and the +# progress commit touches this job's DB session, which is not thread-safe. +_RESOLUTION_CHUNK_SIZE = 200 + + +class _BestEffortReference: + """Wraps the UTA transcript-reference port so a mid-run failure disables reference-identical + detection for the affected input (returns ``None``) instead of failing the VEP job. + + Reference-identical labelling is a refinement for wild-type controls; it must never take down + resolution. A lookup that raises (a dropped UTA connection, a query error) is logged once and + treated as "cannot decide", so the input falls back to ``ABSENT`` exactly as it would with no + reference port. Runs on the resolution executor thread, so it logs without ``extra`` context. + """ + + def __init__(self, inner: ReferenceSequence) -> None: + self._inner = inner + self._warned = False + + def coding_interval_reference(self, transcript: str, start: int, stop: int) -> Optional[str]: + try: + return self._inner.coding_interval_reference(transcript, start, stop) + except Exception as exc: # noqa: BLE001 - any UTA failure degrades to no reference-identical label + if not self._warned: + logger.warning("UTA reference lookup failed mid-run; reference-identical detection degraded: %s", exc) + self._warned = True + return None + + +@contextlib.contextmanager +def _reference_source_for_vep(job_manager: JobManager) -> Iterator[Optional[ReferenceSequence]]: + """Yield a transcript-reference port for reference-identical detection, or ``None`` when UTA is + unavailable. + + Best-effort by design: VEP resolution must not fail because UTA is unset or unreachable. The port is + consulted only for no-change controls and unparseables — everything VEP resolves normally is + untouched — so without it those inputs simply fall back to ``ABSENT`` rather than being labelled + ``REFERENCE_IDENTICAL``. An open-time failure is caught here; a mid-run failure is caught by + :class:`_BestEffortReference`. + """ + with contextlib.ExitStack() as stack: + try: + source = stack.enter_context(uta_transcript_source()) + except Exception as exc: # noqa: BLE001 - opening UTA failed; run without reference-identical + logger.warning( + msg=f"UTA transcript source unavailable; reference-identical detection disabled for this run " + f"(wild-type controls resolve to ABSENT). Cause: {exc}", + extra=job_manager.logging_context(), + ) + yield None + return + yield _BestEffortReference(source) + + +def _annotate_vep( + annotation_manager: AnnotationStatusManager, + allele_id: int, + disposition: Disposition, + reason: EventReason, + *, + source_version: str, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one VEP_FUNCTIONAL_CONSEQUENCE event for an allele (the consequence is an allele-level fact). + + The single choke point for VEP's status writes. One event per allele, stamped with the Ensembl + release queried; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. The consequence value itself is not embedded — it lives in + the ``VepAlleleConsequence`` value table, joinable by ``allele_id`` + ``source_version``. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) + + +def _vep_input_for_allele(row: ScoreSetAlleleRow) -> VepInput | None: + """Build a VEP input for an allele, or ``None`` to skip it. + + Only coding and genomic alleles are submitted to VEP: a protein allele's consequence is carried + instead by the coding alleles reverse translation enumerates from it, so protein alleles are never + sent. Each allele carries exactly one of ``hgvs_g``/``hgvs_c``/``hgvs_p``. + + No transcript is set on the input. A coding allele carries its transcript inside the ``c.`` HGVS + accession, which the kernel infers, so it resolves against its own transcript (the #772 fix). A + genomic allele carries only an ``NC_`` chromosome accession, so it resolves to VEP's + cross-transcript ``most_severe`` headline — deliberately, not as a gap. Alleles are deduplicated + and content-addressed, so one genomic allele is shared across every score set that maps to it; + there is no single "assay transcript" to attribute to it, and borrowing the coding transcript from + one arbitrary pairing would be less faithful than the transcript-agnostic ``most_severe`` call. + ``most_severe`` is the honest reconstruction for a genomic allele, and its + ``consequence_source='most_severe'`` records that provenance so consumers can tell it apart from a + transcript-matched call. + """ + if row.level == SequenceLevel.protein.value: + return None + + hgvs = row.hgvs_c or row.hgvs_g + if not hgvs: + return None + + return VepInput(hgvs=hgvs) @with_pipeline_management async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate VEP functional consequence predictions for all mapped variants in a ScoreSet. + """Link deduplicated alleles to their VEP molecular consequence. - This function retrieves all mapped variants with a populated hgvs_assay_level field for a given - ScoreSet and submits them to the Ensembl VEP API in configurable batches. It handles fallback - to the Variant Recoder API for variants that cannot be processed by VEP directly. + Runs over the score set's current coding and genomic alleles (authoritative and RT-derived), + resolves each against its own transcript via the shared kernel (with a Variant Recoder fallback), + and stores the consequence plus its resolution provenance in a valid-time + :class:`VepAlleleConsequence`, superseding only on a changed headline term. Protein alleles are not + submitted — their consequence is carried by the coding alleles reverse translation enumerates. + + Wild-type controls (an input describing no sequence change) are resolved to a ``reference_identical`` + consequence via a best-effort UTA transcript-reference port, so a control is stored distinctly rather + than dropped as a no-result. If UTA is unavailable the run proceeds without that label (those alleles + fall back to absent); see :func:`_reference_source_for_vep`. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the current-release/resolver skip and re-query every eligible + allele. The linker still supersedes only on a headline-term change, so a forced re-run of + unchanged data writes no new rows. Use for re-ingestion or to heal suspected corruption. A + resolution-rule change no longer needs force: it bumps RESOLVER_VERSION, which the skip keys + on, so those alleles re-query on the next ordinary run. Args: ctx (dict): The job context dictionary. @@ -52,7 +190,9 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager (JobManager): Manager for job lifecycle and DB operations. Returns: - JobExecutionOutcome: Outcome with counts of processed, successful, and failed variants. + JobExecutionOutcome: outcome with per-allele created/preexisting/absent/errored counts, plus + ``retained_on_absence_count`` — preexisting alleles whose consequence was kept because VEP + returned nothing this run (a genuine-disappearance signal, surfaced in the job metadata). """ job = job_manager.get_job() @@ -62,6 +202,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Safely ignore mypy warnings here, as params were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] job_manager.save_to_context( { @@ -71,348 +212,208 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan "correlation_id": correlation_id, } ) - job_manager.update_progress(0, 100, "Starting VEP population.") - logger.info(msg="Started VEP population", extra=job_manager.logging_context()) - - mapped_variants = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.post_mapped.isnot(None), - ) - ).all() - - if not mapped_variants: - logger.warning( - msg=f"No mapped variants found for score set {score_set.urn}. Skipped VEP population.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "variants_processed": 0, - "variants_with_consequences": 0, - "variants_without_consequences": 0, - "variants_recoder_failed": 0, - } - ) - - job_manager.save_to_context({"total_variants_to_process": len(mapped_variants)}) - logger.info( - msg=f"Found {len(mapped_variants)} mapped variants for VEP processing", - extra=job_manager.logging_context(), + job_manager.update_progress(0, 100, "Starting VEP consequence linkage.") + logger.info(msg="Started VEP consequence linkage", extra=job_manager.logging_context()) + + # One work-unit per eligible allele (payload = a transcript-aware VepInput; protein alleles and + # alleles without HGVS are skipped). Events are allele-keyed, so each allele records its own event. + allele_inputs = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=_vep_input_for_allele, ) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - mapped_variants_by_id = {mv.id: mv for mv in mapped_variants} - - # Extract HGVS strings; skip and annotate variants that have none. - hgvs_and_mapped_variant_id_pairs: list[tuple[str, int]] = [] - - for mapped_variant in mapped_variants: - if not mapped_variant.hgvs_assay_level: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={"error_message": "Mapped variant does not have an associated HGVS string."}, - ) - logger.debug("Mapped variant does not have an associated HGVS string.", extra=job_manager.logging_context()) - continue - - hgvs_and_mapped_variant_id_pairs.append((mapped_variant.hgvs_assay_level, mapped_variant.id)) # type: ignore - - batches = list(batched(hgvs_and_mapped_variant_id_pairs, _VEP_BATCH_SIZE)) - - job_manager.save_to_context({"vep_batches": len(batches)}) - logger.debug( - msg=f"Prepared {len(batches)} VEP batches ({_VEP_BATCH_SIZE} variants/batch)", - extra=job_manager.logging_context(), + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "absent_allele_count": 0, + "errored_allele_count": 0, + # A subset of preexisting: alleles that kept a prior consequence because VEP returned nothing + # this run. Surfaced in the job outcome so a genuine-disappearance pattern is visible without + # digging through logs (see VepLinkVerdict.RETAINED_ON_ABSENCE). + "retained_on_absence_count": 0, + } ) - # --- Phase 1: Initial VEP pass --- - all_consequences: dict[str, str] = {} - all_missing_hgvs: set[str] = set() + num_alleles_with_hgvs = len(allele_inputs) + job_manager.save_to_context({"num_alleles_to_link_vep": num_alleles_with_hgvs}) - for batch_idx, batch in enumerate(batches): - logger.debug( - msg=f"Processing VEP batch {batch_idx + 1}/{len(batches)}", - extra=job_manager.logging_context(), - ) - - hgvs_strings, mapped_variant_ids = map(list, zip(*batch)) # type: ignore - - consequences = await get_functional_consequence(hgvs_strings) - logger.debug( - msg=f"Received consequences for {len(consequences)} variants in VEP batch {batch_idx + 1}", + if not allele_inputs: + logger.warning( + msg="No current coding/genomic alleles with HGVS were found for this score set. Skipping VEP linkage (nothing to do).", extra=job_manager.logging_context(), ) - - # Only store variants where VEP returned an actual consequence string. A None value - # means VEP knew the variant but couldn't classify it — treat that the same as absent - # and route to Recoder so we have the best chance of getting a consequence. - hit_consequences = {h: c for h, c in consequences.items() if c is not None} - all_consequences.update(hit_consequences) - - missing_hgvs = set(hgvs_strings) - set(hit_consequences.keys()) - for hgvs, mapped_variant_id in zip(hgvs_strings, mapped_variant_ids): - if hgvs in missing_hgvs: - all_missing_hgvs.add(hgvs) - - progress_pct = int((batch_idx + 1) / len(batches) * 33) - job_manager.save_to_context( - { - "initial_vep_batches_processed": batch_idx + 1, - "missing_hgvs_count": len(all_missing_hgvs), - } - ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", + job_manager.db.flush() + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) + + all_allele_ids = set(allele_inputs.keys()) + + # The Ensembl release version-keys the run (coordinated software + transcript set + vocabulary). It + # is load-bearing for the skip below, so a failure here aborts the job rather than mis-versioning + # writes (the exception propagates to the job decorators). + ensembl_release = await get_ensembl_release() + job_manager.save_to_context({"ensembl_release": ensembl_release}) + + def alleles_at_current_release(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live VEP consequence at the current Ensembl release + *and* the current resolver version — both axes must match to count as up to date.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(VepAlleleConsequence.allele_id) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.current) + .where(VepAlleleConsequence.functional_consequence.isnot(None)) + .where(VepAlleleConsequence.source_version == ensembl_release) + # A resolution-rule change bumps RESOLVER_VERSION without an Ensembl release bump; gating + # on both axes re-queries those alleles automatically instead of them looking current + # forever. A NULL resolver_version (pre-column row) never matches and so is re-queried. + .where(VepAlleleConsequence.resolver_version == RESOLVER_VERSION) + ).all() ) - logger.info( - msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} variants require Variant Recoder fallback.", - extra=job_manager.logging_context(), + # Skip alleles already resolved at the current Ensembl release AND resolver version (they cannot + # change without one of those bumping). force re-queries all regardless — for re-ingestion or to heal + # suspected corruption. The linker still supersedes only on a value change, so a forced (or + # resolver-bumped) no-op advances the version in place and writes no new history row. + already_current = set() if force else alleles_at_current_release(all_allele_ids) + inputs_by_allele = {aid: allele_inputs[aid] for aid in allele_inputs if aid not in already_current} + # HGVS alone identifies a unique resolution question here: this job never sets VepInput.transcript + # (a coding allele's transcript rides inside its c. HGVS; a genomic allele is transcript-agnostic — + # see _vep_input_for_allele), and resolve_consequences re-keys its result by HGVS to match. If this + # job ever passes explicit transcripts, both this dedup key and that return keying must become + # (hgvs, transcript), or same-HGVS/different-transcript inputs collapse into one. + unique_inputs = list({i.hgvs: i for i in inputs_by_allele.values()}.values()) + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_hgvs_to_query": len(unique_inputs), + "force": force, + } ) - # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- - hgvs_to_genomic: dict[str, list[str]] = {} - recoder_missing_hgvs: set[str] = set() - - if all_missing_hgvs: - logger.info( - msg=f"Running Variant Recoder for {len(all_missing_hgvs)} HGVS strings", - extra=job_manager.logging_context(), - ) - - recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) - - logger.debug( - msg=f"Running {len(recoder_batch_list)} Variant Recoder batches with concurrency {_RECODER_CONCURRENCY}", - extra=job_manager.logging_context(), - ) - - semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) - completed_recoder_batches = 0 - - async def _recoder_with_semaphore(batch: list[str], batch_idx: int, total: int) -> dict[str, list[str]]: - nonlocal completed_recoder_batches - async with semaphore: - logger.debug( - msg=f"Starting Variant Recoder batch {batch_idx + 1}/{total} ({len(batch)} HGVS strings)", - extra=job_manager.logging_context(), - ) - result = await run_variant_recoder(batch) - completed_recoder_batches += 1 - logger.debug( - msg=f"Completed Variant Recoder batch {completed_recoder_batches}/{total} ({len(result)} variants recoded)", - extra=job_manager.logging_context(), - ) - progress_pct = 33 + int(completed_recoder_batches / total * 33) + verdicts: dict[int, VepLinkVerdict] = {} + errored_allele_ids: set[int] = set() + if unique_inputs: + # The shared orchestration exposes no per-input progress hook, so resolve in chunks and report + # between them. Progress is emitted here on the coroutine, not from inside resolve_consequences: + # that runs on an executor thread and the progress commit writes this job's (non-thread-safe) DB + # session. Chunk-level granularity is the resolution; finer would require a kernel progress hook. + job_manager.update_progress(10, 100, f"Resolving VEP consequences for {len(unique_inputs)} HGVS strings.") + resolutions_by_hgvs: dict[str, ConsequenceResolution] = {} + chunks = list(batched(unique_inputs, _RESOLUTION_CHUNK_SIZE)) + # A no-change control (an explicit c.= form, or a delins whose bases equal the transcript's own) + # is not a VEP-resolvable variant; the reference port lets it resolve to REFERENCE_IDENTICAL + # instead of vanishing into ABSENT. Best-effort: a missing/down UTA just disables that label. + with _reference_source_for_vep(job_manager) as reference: + for chunk_idx, chunk in enumerate(chunks): + resolutions_by_hgvs.update(await resolve_consequences(list(chunk), reference=reference)) job_manager.update_progress( - progress_pct, + 10 + int((chunk_idx + 1) / len(chunks) * 85), 100, - f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", + f"Resolved VEP consequences for batch {chunk_idx + 1}/{len(chunks)}.", ) - return result - - total_recoder_batches = len(recoder_batch_list) - recoder_results = await asyncio.gather( - *[ - _recoder_with_semaphore(list(recoder_batch), idx, total_recoder_batches) - for idx, recoder_batch in enumerate(recoder_batch_list) - ], - return_exceptions=True, - ) - - successful_batches = sum(1 for r in recoder_results if not isinstance(r, Exception)) - first_exception = next((r for r in recoder_results if isinstance(r, Exception)), None) - if first_exception is not None: - logger.error( - msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {str(first_exception)}", - extra=job_manager.logging_context(), - ) - raise first_exception - - for result in recoder_results: - hgvs_to_genomic.update(result) # type: ignore[arg-type] + # Every queried input gets a resolution. An allele missing a resolution is treated as genuinely empty + # and never a failure. This ensures a dropped input can only under-report, never overwrite a held consequence. + resolution_by_allele = { + aid: resolutions_by_hgvs.get(inp.hgvs) + or ConsequenceResolution(input=inp, outcome=ConsequenceOutcome.ABSENT) + for aid, inp in inputs_by_allele.items() + } - job_manager.save_to_context( - { - "variant_recoder_batches_processed": len(recoder_batch_list), - "recoded_variants_count": len(hgvs_to_genomic), - } + # Alleles whose VEP/Recoder request failed: unknown, not a negative — kept distinct from a genuine + # empty. Never linked, since that would overwrite a held consequence with a failure. + errored_allele_ids = { + aid for aid, res in resolution_by_allele.items() if res.outcome is ConsequenceOutcome.ERRORED + } + linkable = { + aid: res for aid, res in resolution_by_allele.items() if res.outcome is not ConsequenceOutcome.ERRORED + } + verdicts = link_vep_consequences_to_alleles( + job_manager.db, linkable, source_version=ensembl_release, access_date=date.today() ) + job_manager.db.flush() + else: logger.info( - msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} variants successfully recoded.", + msg="All eligible alleles are already resolved at the current Ensembl release; skipping VEP query.", extra=job_manager.logging_context(), ) + job_manager.update_progress(99, 100, "All alleles already current at this Ensembl release.") - # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- - # hgvs_to_genomic maps original HGVS → list[str]; flatten to a deduplicated list of - # genomic strings before batching with VEP. - all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) - recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) - all_recoded_consequences: dict[str, str | None] = {} - - for recoded_vep_batch_idx, recoded_vep_batch in enumerate(recoded_vep_batch_list): - logger.debug( - msg=f"Processing recoded HGVS VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", - extra=job_manager.logging_context(), + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, vep_input in allele_inputs.items(): + verdict = verdicts.get(allele_id) + if verdict is VepLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + source_version=ensembl_release, + metadata={"hgvs": vep_input.hgvs}, ) - recoded_vep_consequences = await get_functional_consequence(recoded_vep_batch) - all_recoded_consequences.update(recoded_vep_consequences) - - progress_pct = 66 + int((recoded_vep_batch_idx + 1) / len(recoded_vep_batch_list) * 33) - job_manager.save_to_context( - { - "recoded_vep_batches_processed": recoded_vep_batch_idx + 1, - "recoded_consequences_count": len(all_recoded_consequences), - } - ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed recoded VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", + elif allele_id in already_current or verdict in ( + VepLinkVerdict.UNCHANGED, + VepLinkVerdict.RETAINED_ON_ABSENCE, + ): + # The allele still has a live consequence, so its status is preexisting either way. When it + # was retained despite VEP finding nothing this run, also tally it separately for the outcome. + annotation_counts["preexisting_allele_count"] += 1 + if verdict is VepLinkVerdict.RETAINED_ON_ABSENCE: + annotation_counts["retained_on_absence_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=ensembl_release, + metadata={"hgvs": vep_input.hgvs}, ) - logger.info( - msg=f"Completed recoded VEP processing. {len(all_recoded_consequences)} recoded consequences retrieved.", - extra=job_manager.logging_context(), - ) - - # Map most-severe consequence from recoded genomic HGVS back to the original HGVS. - for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): - recoded_consequences_for_variant = [ - c for recoded_hgvs in recoded_hgvs_list if (c := all_recoded_consequences.get(recoded_hgvs)) - ] - - if recoded_consequences_for_variant: - most_severe = next( - (c for c in VEP_CONSEQUENCES if c in recoded_consequences_for_variant), - None, - ) - if most_severe: - all_consequences[original_hgvs] = most_severe - logger.debug( - msg=f"Selected most severe consequence '{most_severe}' for {original_hgvs}", - extra=job_manager.logging_context(), - ) - else: - logger.debug( - msg=f"Could not retrieve functional consequences for any recoded variants of {original_hgvs}", - extra=job_manager.logging_context(), - ) - - recoder_missing_hgvs = all_missing_hgvs - set(hgvs_to_genomic.keys()) - - # --- Phase 4: Annotate outcomes and update mapped variants in a single pass --- - - # HGVS strings that went through both VEP passes but still have no consequence. - all_processed_hgvs = {h for h, _ in hgvs_and_mapped_variant_id_pairs} - vep_failed_hgvs = all_processed_hgvs - set(all_consequences.keys()) - recoder_missing_hgvs - - variants_processed = 0 - variants_with_consequences = 0 - variants_without_consequences = 0 - variants_recoder_failed = 0 - - for hgvs_string, mapped_variant_id in hgvs_and_mapped_variant_id_pairs: - mapped_variant = mapped_variants_by_id.get(mapped_variant_id) # type: ignore - if mapped_variant is None: - continue - - consequence = all_consequences.get(hgvs_string) - if consequence: - mapped_variant.vep_functional_consequence = consequence - mapped_variant.vep_access_date = date.today() - job_manager.db.add(mapped_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"functional_consequence": consequence}}, - ) - variants_with_consequences += 1 - logger.debug( - msg=f"Set consequence '{consequence}' for mapped variant {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in vep_failed_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "VEP could not determine a functional consequence for this variant, even after Variant Recoder fallback.", - }, - ) - variants_without_consequences += 1 - logger.debug( - msg=f"Recorded VEP failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in recoder_missing_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "Variant Recoder could not recode this HGVS string to a genomic equivalent.", - }, - ) - variants_recoder_failed += 1 - logger.debug( - msg=f"Recorded Variant Recoder failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), + elif allele_id in errored_allele_ids: + annotation_counts["errored_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=ensembl_release, + error_message="The VEP/Variant Recoder request for this allele failed; result unknown.", + metadata={"hgvs": vep_input.hgvs}, ) + else: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": "Variant was not classified by any VEP outcome branch. This is a bug.", - }, - ) - variants_without_consequences += 1 - logger.warning( - msg=f"Unexpected state: mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string}) was not classified by any outcome branch.", - extra=job_manager.logging_context(), + annotation_counts["absent_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=ensembl_release, + error_message="VEP found no functional consequence for this allele, even after Variant Recoder fallback.", + metadata={"hgvs": vep_input.hgvs}, ) - variants_processed += 1 - annotation_manager.flush() - job_manager.db.flush() + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) job_manager.update_progress( 100, 100, - f"Completed VEP functional consequence prediction for {variants_with_consequences}/{variants_processed} variants.", - ) - logger.info( - msg=f"Completed VEP prediction: {variants_with_consequences} with consequences, {variants_without_consequences} without, {variants_recoder_failed} recoder failed", - extra=job_manager.logging_context(), + ( + f"Completed VEP linkage: {annotation_counts['created_allele_count'] + annotation_counts['preexisting_allele_count']} linked, " + f"{annotation_counts['absent_allele_count']} absent (no result), " + f"{annotation_counts['errored_allele_count']} errored, " + f"{annotation_counts['retained_on_absence_count']} retained despite no result this run." + ), ) - + logger.info(msg="Done linking VEP consequences to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "variants_processed": variants_processed, - "variants_with_consequences": variants_with_consequences, - "variants_without_consequences": variants_without_consequences, - "variants_recoder_failed": variants_recoder_failed, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index 94cd76e32..33878de9e 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -18,8 +18,6 @@ from mavedb.worker.jobs.external_services import ( link_gnomad_variants, poll_uniprot_mapping_jobs_for_score_set, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, refresh_clinvar_controls, submit_score_set_mappings_to_car, @@ -32,6 +30,7 @@ from mavedb.worker.jobs.variant_processing import ( create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, ) # All job functions for ARQ worker @@ -39,6 +38,7 @@ # Variant processing jobs create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, # External service jobs submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -47,8 +47,6 @@ submit_uniprot_mapping_jobs_for_score_set, poll_uniprot_mapping_jobs_for_score_set, link_gnomad_variants, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, # Data management jobs refresh_materialized_views, @@ -71,8 +69,10 @@ cron( cleanup_stalled_jobs, name="cleanup_stalled_jobs_cron", - minute={15, 45}, # Run at :15 and :45 past each hour (every 30 minutes) - keep_result=timedelta(minutes=25).total_seconds(), + # Every 10 minutes. Paired with the 30-min progress-stall window this gives an effective + # stall-detection latency of ~30-40 min. + minute={5, 15, 25, 35, 45, 55}, + keep_result=timedelta(minutes=9).total_seconds(), ), ] @@ -100,6 +100,13 @@ "key": "map_variants_for_score_set", "type": JobType.VARIANT_MAPPING, }, + reverse_translate_variants_for_score_set: { + "dependencies": [], + "params": {"score_set_id": None, "correlation_id": None}, + "function": "reverse_translate_variants_for_score_set", + "key": "reverse_translate_variants_for_score_set", + "type": JobType.VARIANT_MAPPING, + }, submit_score_set_mappings_to_car: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, @@ -149,20 +156,6 @@ "key": "link_gnomad_variants", "type": JobType.MAPPED_VARIANT_ANNOTATION, }, - populate_hgvs_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_hgvs_for_score_set", - "key": "populate_hgvs_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, - populate_variant_translations_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_variant_translations_for_score_set", - "key": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, populate_vep_for_score_set: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 89089828c..f44ed9221 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -13,6 +13,7 @@ The cleanup job acts as a safety net to ensure jobs don't remain in limbo forever. """ +import asyncio import logging from datetime import datetime, timedelta, timezone @@ -32,19 +33,35 @@ from mavedb.worker.lib.managers.constants import ACTIVE_JOB_STATUSES, TERMINAL_PIPELINE_STATUSES from mavedb.worker.lib.managers.job_manager import JobManager from mavedb.worker.lib.managers.pipeline_manager import PipelineManager -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name logger = logging.getLogger(__name__) # Timeout thresholds for detecting stalled jobs (in minutes). -# RUNNING_TIMEOUT_MINUTES must stay below ArqWorkerSettings.job_timeout (currently 2 hours) -# to avoid marking legitimately running jobs as stalled. -RUNNING_TIMEOUT_MINUTES = 150 # RUNNING jobs should complete within 150 min (30 min buffer under ARQ timeout) +# +# RUNNING jobs are evaluated on two signals: +# 1. PROGRESS_STALL_MINUTES — the primary signal. A healthy job checkpoints progress frequently +# (JobRun.progress_updated_at is bumped on every committing progress update via onupdate=now()). +# If the heartbeat has not advanced in this window the job is considered wedged, regardless of +# how long it has legitimately been running. This is what lets a multi-hour VEP fan-out run to +# completion without being killed, while still catching a job that has genuinely hung. +# 2. RUNNING_TIMEOUT_MINUTES — a wall-clock backstop for jobs that never emit progress (or whose +# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (24h) so +# the DB-driven sweeper, not ARQ's hard kill, is what terminates a job — keeping DB state and +# actual work in sync. +PROGRESS_STALL_MINUTES = 30 # RUNNING jobs should checkpoint progress at least this often +RUNNING_TIMEOUT_MINUTES = 1410 # Wall-clock backstop (23.5h), 30 min under the 24h ARQ ceiling PENDING_TIMEOUT_MINUTES = 5 # PENDING jobs which are actionable within pipelines should be enqueued within 5 minutes PIPELINE_STUCK_TIMEOUT_MINUTES = ( 5 # Pipelines in non-terminal states with no active jobs should resolve within 5 minutes ) +# How long the sweeper waits for an aborted RUNNING job to actually die before giving up. A job +# cancelled at an await point dies within a poll cycle (sub-second); a job wedged in a blocking sync +# call never honors the abort and will hit this timeout — in which case we do NOT re-drive it (see +# _handle_stalled_running_job) to avoid running it twice. +ABORT_CONFIRM_TIMEOUT_SECONDS = 15 + async def _handle_stalled_job_retry( job: JobRun, @@ -162,7 +179,9 @@ async def _handle_stalled_job_retry( try: manager.prepare_queue() # Transition to QUEUED db.flush() - result = await redis.enqueue_job(job.job_function, job.id, _job_id=arq_job_id(job)) + result = await redis.enqueue_job( + job.job_function, job.id, _job_id=arq_job_id(job), _queue_name=arq_queue_name(job) + ) if result is None: raise RuntimeError( @@ -193,19 +212,90 @@ async def _handle_stalled_job_retry( return False +async def _running_job_confirmed_dead(arq_id: str, queue_name: str, redis: ArqRedis) -> bool: + """Abort a RUNNING job's current ARQ attempt and report whether it is safe to re-drive. + + RUNNING recovery differs from QUEUED/PENDING recovery: a RUNNING job may still have a live + coroutine holding a worker slot. Re-enqueuing without cancelling it first may mean two + coroutines are running the same job simultaneously, which is unsafe. + + We therefore issue Job.abort() on the current attempt and interpret the outcome: + + - abort() returns True -> the coroutine was alive and has now been cancelled + recorded a + terminal result. Safe to re-drive. + - abort() returns False -> the job is absent from ARQ (crashed worker / never enqueued / already + finished). Nothing is running. Safe to re-drive — this is the common crash-recovery path. + - abort() raises TimeoutError -> the job is still present in ARQ and did not die within the + window: a coroutine wedged in a blocking sync call that cannot honor cooperative cancellation. + NOT safe to re-drive; leave it for the next sweep or ARQ's own ceiling. + - any other error -> treat as not-confirmably-alive and allow re-drive; a genuinely live job + would have either confirmed the abort or timed out, not errored here. + + Returns True when it is safe to re-drive the job, False when the job may still be running. + """ + try: + await ArqJob(arq_id, redis, _queue_name=queue_name).abort(timeout=ABORT_CONFIRM_TIMEOUT_SECONDS) + return True + except asyncio.TimeoutError: + logger.warning( + f"Abort of RUNNING job attempt {arq_id} did not confirm within " + f"{ABORT_CONFIRM_TIMEOUT_SECONDS}s; job may be wedged in a blocking call — not re-driving" + ) + return False + except Exception as e: + logger.warning(f"Abort of RUNNING job attempt {arq_id} raised {e!r}; treating as not running and re-driving") + return True + + +async def _handle_stalled_running_job( + job: JobRun, + manager: JobManager, + redis: ArqRedis, + stall_reason: str, + db: Session, +) -> bool: + """Recover a stalled RUNNING job via abort-first ordering. + + Aborts the current attempt (urn#N) BEFORE any retry bookkeeping, and only proceeds to re-drive + (which increments retry_count and enqueues urn#N+1) once the abort confirms the original is dead + or absent. If the original refuses to die, we leave it alone rather than risk a double-run. + + Returns True if the sweeper acted on the job (re-drove or permanently failed it), False if it was + left in place because the abort could not be confirmed (so it should not be counted as cleaned). + """ + arq_id = arq_job_id(job) + + # Could not confirm the coroutine is dead. Leave the job RUNNING; the next sweep will retry + # the abort, and ARQ's job_timeout ceiling remains as a final backstop. + if not await _running_job_confirmed_dead(arq_id, arq_queue_name(job), redis): + logger.warning( + f"Leaving stalled RUNNING job {job.urn} in place — abort unconfirmed, re-driving would risk a double-run", + extra=manager.logging_context(), + ) + return False + + # Abort confirmed dead (or the job was absent from ARQ). Refresh our view — if the aborted + # coroutine's own lifecycle handler already marked the row terminal, we want to see that — then + # re-drive through the shared retry handler, which preserves pipeline dependency semantics. + db.refresh(job) + await _handle_stalled_job_retry(job, manager, redis, stall_reason, db) + return True + + @with_guaranteed_job_run_record("cron_job") @with_job_management async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """Detect and handle jobs that have stalled in intermediate states. - This job runs periodically (every 15 minutes) to find jobs that have been - stuck in QUEUED, RUNNING, or PENDING states beyond reasonable timeouts - and handles them appropriately. + This job runs periodically to find jobs that have been stuck in QUEUED, RUNNING, or PENDING + states beyond reasonable timeouts and handles them appropriately. Stalled job detection criteria: - QUEUED: Present in DB as QUEUED but absent from ARQ's Redis queue (process crashed between prepare_queue and redis.enqueue_job) - - RUNNING: Started > 60 minutes ago but not finished (worker likely crashed) + - RUNNING: Progress heartbeat idle > PROGRESS_STALL_MINUTES, or wall-clock runtime past the + RUNNING_TIMEOUT_MINUTES backstop, and not finished. Recovered abort-first (see + _handle_stalled_running_job) so a wedged attempt is never re-driven alongside its own retry. - PENDING: Created > 5 minutes ago in a pipeline and currently runnable (coordination failure) - Pipeline stuck: Non-terminal pipeline with no active jobs older than 5 minutes @@ -230,7 +320,8 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) Job stalled in RUNNING (worker crash): - Worker started job, marked it RUNNING, then crashed - - After 60 minutes (longer than ARQ timeout), janitor detects and retries + - Once the progress heartbeat goes stale (or the wall-clock backstop is hit), the janitor + aborts the dead attempt and re-drives it """ job_manager.save_to_context( { @@ -280,7 +371,7 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) # A healthy job waiting for a worker slot appears QUEUED in the DB and is also # present in Redis; only a crashed-enqueue job has the DB state without the # corresponding Redis entry. - arq_status = await ArqJob(arq_job_id(job), job_manager.redis).status() + arq_status = await ArqJob(arq_job_id(job), job_manager.redis, _queue_name=arq_queue_name(job)).status() if arq_status in (ArqJobStatus.queued, ArqJobStatus.in_progress, ArqJobStatus.deferred): logger.debug( f"QUEUED job {job.urn} is present in ARQ Redis (status={arq_status.value}); skipping cleanup", @@ -305,14 +396,19 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) job_manager.save_to_context({"cleaned_queued_jobs": queued_jobs}) logger.debug("Completed cleaning stalled QUEUED jobs.", extra=job_manager.logging_context()) - # Find RUNNING jobs that have been running too long OR have missing started_at - # These likely indicate worker crashes (worker died mid-execution) or data inconsistencies + # Find RUNNING jobs that are wedged. A job is a candidate when EITHER its progress heartbeat has + # gone stale (the primary, runtime-agnostic signal) OR it has blown past the wall-clock backstop + # (or is missing started_at entirely — a data inconsistency). Healthy long-running jobs keep + # bumping progress_updated_at and so are never selected until the backstop. + progress_stall_threshold = now - timedelta(minutes=PROGRESS_STALL_MINUTES) running_threshold = now - timedelta(minutes=RUNNING_TIMEOUT_MINUTES) running_jobs = job_manager.db.scalars( select(JobRun).where( JobRun.status == JobStatus.RUNNING, - (JobRun.started_at < running_threshold) | (JobRun.started_at.is_(None)), JobRun.finished_at.is_(None), + (JobRun.progress_updated_at < progress_stall_threshold) + | (JobRun.started_at < running_threshold) + | (JobRun.started_at.is_(None)), ) ).all() @@ -333,18 +429,26 @@ async def cleanup_stalled_jobs(ctx: dict, job_id: int, job_manager: JobManager) continue elapsed_minutes = (now - job.started_at).total_seconds() / 60 + heartbeat_stale = job.progress_updated_at < progress_stall_threshold + if heartbeat_stale: + heartbeat_age = (now - job.progress_updated_at).total_seconds() / 60 + stall_reason = ( + f"Job stalled in RUNNING state: progress heartbeat idle for {heartbeat_age:.1f} minutes " + f"(running for {elapsed_minutes:.1f} minutes)" + ) + else: + stall_reason = ( + f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes " + f"(exceeded {RUNNING_TIMEOUT_MINUTES} min backstop; likely worker crash)" + ) - logger.warning( - f"Detected stalled RUNNING job {job.urn} " - f"(started {job.started_at}, running for {elapsed_minutes:.1f} minutes)", - extra=manager.logging_context(), - ) + logger.warning(f"Detected stalled RUNNING job {job.urn}: {stall_reason}", extra=manager.logging_context()) - stall_reason = f"Job stalled in RUNNING state for {elapsed_minutes:.1f} minutes (likely worker crash)" - await _handle_stalled_job_retry(job, manager, job_manager.redis, stall_reason, job_manager.db) + acted = await _handle_stalled_running_job(job, manager, job_manager.redis, stall_reason, job_manager.db) manager.db.commit() - cleaned_jobs["running"].append(job.urn) + if acted: + cleaned_jobs["running"].append(job.urn) job_manager.save_to_context({"cleaned_running_jobs": running_jobs}) logger.debug("Completed cleaning stalled RUNNING jobs.", extra=job_manager.logging_context()) diff --git a/src/mavedb/worker/jobs/variant_processing/__init__.py b/src/mavedb/worker/jobs/variant_processing/__init__.py index a6df09753..240de5e28 100644 --- a/src/mavedb/worker/jobs/variant_processing/__init__.py +++ b/src/mavedb/worker/jobs/variant_processing/__init__.py @@ -10,8 +10,10 @@ from .mapping import ( map_variants_for_score_set, ) +from .reverse_translation import reverse_translate_variants_for_score_set __all__ = [ "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", ] diff --git a/src/mavedb/worker/jobs/variant_processing/creation.py b/src/mavedb/worker/jobs/variant_processing/creation.py index a519bda07..333f167f0 100644 --- a/src/mavedb/worker/jobs/variant_processing/creation.py +++ b/src/mavedb/worker/jobs/variant_processing/creation.py @@ -13,14 +13,24 @@ from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, RESTDataProvider, s3_client from mavedb.lib.logging.context import format_raised_exception_info_as_dict +from mavedb.lib.score_calibrations import ( + restore_calibration_variant_links, + snapshot_calibration_variant_links, +) +from mavedb.lib.score_distribution import summarize_variant_data from mavedb.lib.score_sets import columns_for_dataset, create_variants, create_variants_data +from mavedb.lib.types.score_calibrations import CalibrationVariantLinkSnapshot from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.validation.dataframe.dataframe import validate_and_standardize_dataframe_pair from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.calibration_control import CalibrationControl from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.mapping_state import MappingState from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) from mavedb.models.score_set import ScoreSet from mavedb.models.user import User from mavedb.models.variant import Variant @@ -180,10 +190,30 @@ async def create_variants_for_score_set(ctx: dict, job_id: int, job_manager: Job } # Delete variants after validation occurs so we don't overwrite them in the case of a bad update. + calibration_link_snapshot: list[CalibrationVariantLinkSnapshot] = [] + relink_calibrations = False if score_set.variants: existing_variants = job_manager.db.scalars( select(Variant.id).where(Variant.score_set_id == score_set.id) ).all() + + # Calibration controls and functional classification bin membership reference variants + # without an ON DELETE action, so the variant delete below fails outright while those + # links stand. Record what the new upload cannot regenerate, clear the links, and + # re-establish them once the new variants exist. + relink_calibrations = bool(score_set.score_calibrations) + calibration_link_snapshot = snapshot_calibration_variant_links(job_manager.db, score_set) + job_manager.db.execute( + delete(CalibrationControl).where(CalibrationControl.variant_id.in_(existing_variants)) + ) + job_manager.db.execute( + delete(score_calibration_functional_classification_variants_association_table).where( + score_calibration_functional_classification_variants_association_table.c.variant_id.in_( + existing_variants + ) + ) + ) + job_manager.db.execute(delete(MappedVariant).where(MappedVariant.variant_id.in_(existing_variants))) job_manager.db.execute(delete(Variant).where(Variant.id.in_(existing_variants))) @@ -196,8 +226,26 @@ async def create_variants_for_score_set(ctx: dict, job_id: int, job_manager: Job job_manager.db.refresh(score_set) variants_data = create_variants_data(validated_scores, validated_counts, None) + score_set.score_distribution = summarize_variant_data(variant["data"] for variant in variants_data) create_variants(job_manager.db, score_set, variants_data) + # Scoped to the replacement path above: only links the delete broke need re-establishing, so a + # first upload leaves calibrations alone. An empty snapshot is not the same as nothing to do, + # though, since range-based bins are recomputed from the new scores rather than remembered. + if relink_calibrations: + # The new variants need ids and scores on record before anything can point at them. + job_manager.db.flush() + + relink_report = restore_calibration_variant_links( + job_manager.db, score_set, calibration_link_snapshot, updated_by + ) + job_manager.save_to_context(relink_report.to_dict()) + + logger.info( + msg="Re-established calibration variant references against the newly created variants.", + extra=job_manager.logging_context(), + ) + except ValidationError as e: job_manager.db.rollback() diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index efc8226b3..9376b59e2 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -6,12 +6,14 @@ """ import asyncio +import contextlib import functools import logging -from datetime import date +from collections import Counter +from datetime import date, datetime, timezone from typing import Any -from sqlalchemy import cast, null, select +from sqlalchemy import cast, func, null, select from sqlalchemy.dialects.postgresql import JSONB from mavedb.data_providers.services import vrs_mapper @@ -23,13 +25,19 @@ ) from mavedb.lib.logging.context import format_raised_exception_info_as_dict from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS +from mavedb.lib.mapping.schema import MappingOutcome from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.lib.vrs_utils import canonical_variation_document, verify_allele_refget +from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.user import User @@ -40,6 +48,48 @@ logger = logging.getLogger(__name__) +# The score-set map is a single opaque blocking call to dcd-mapping: it cannot refresh the progress +# heartbeat mid-call, so `cleanup_stalled_jobs` (PROGRESS_STALL_MINUTES = 30) would reap a live worker +# on any set that maps for longer than that. Two guards bracket the call — a liveness keepalive that +# refreshes the heartbeat on a fixed cadence, and a size-aware wall-clock budget that bounds a wedged +# mapper to a sensible window instead of the 23.5h stall backstop. This is an explicit workaround for +# delegating opaque work; the real fix is for the mapper to report progress (or to chunk the call). +# See worker/best_practices.md, "Long external delegations". TODO: replace with real mapping progress. +MAP_PROGRESS_KEEPALIVE_SECONDS = 300 # heartbeat cadence; must stay well under PROGRESS_STALL_MINUTES (30m) +MAP_BUDGET_BASE_SECONDS = 60 * 60 # floor: two dcd-mapping BLAT runs (BLAT_TIMEOUT_SECONDS, 20 min each) + overhead +MAP_BUDGET_PER_VARIANT_SECONDS = 1.0 # ~3x the observed ~0.3 s/variant map throughput, as slack +MAP_BUDGET_MAX_SECONDS = 22 * 60 * 60 # ceiling: stay under the 23.5h stall backstop / 24h ARQ ceiling + + +async def _keepalive_progress(job_manager: JobManager, interval_seconds: int) -> None: + """Refresh the job's progress heartbeat on a fixed cadence during an opaque, un-checkpointable call. + + Liveness, not progress: the task runs in the worker's event loop, so it dies with a crashed worker + and the heartbeat then correctly goes stale — only a *live* worker in a long delegation is kept from + being falsely reaped. Best-effort; a failed refresh is logged and the loop continues. + """ + while True: + await asyncio.sleep(interval_seconds) + try: + job_manager.update_progress( + 30, + 100, + f"Mapping variants using VRS mapping service (in progress, {datetime.now(timezone.utc):%H:%M:%SZ}).", + ) + except Exception: # noqa: BLE001 - the heartbeat is best-effort; a dropped tick is harmless + logger.warning("Mapping progress keepalive tick failed; continuing.", extra=job_manager.logging_context()) + + +def _map_budget_seconds(variant_count: int) -> int: + """Total wall-clock budget for the opaque map call, sized to the score set. + + Bounds a wedged external mapper to a size-appropriate window instead of the 23.5h stall backstop, + while leaving enough slack (the per-variant term is ~3x observed throughput) not to fail a slow but + healthy map. Clamped to a ceiling below the ARQ job timeout. + """ + budget = MAP_BUDGET_BASE_SECONDS + int(variant_count * MAP_BUDGET_PER_VARIANT_SECONDS) + return min(budget, MAP_BUDGET_MAX_SECONDS) + @with_pipeline_management async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: @@ -102,9 +152,26 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan mapping_results = None + variant_count = job_manager.db.scalar( + select(func.count(Variant.id)).where(Variant.score_set_id == score_set.id) + ) + map_budget = _map_budget_seconds(variant_count or 0) + logger.debug(msg="Mapping variants using VRS mapping service.", extra=job_manager.logging_context()) job_manager.update_progress(30, 100, "Mapping variants using VRS mapping service.") - mapping_results = await loop.run_in_executor(ctx["pool"], blocking) + # Keepalive heartbeat + size-aware budget around the opaque map call (see module docstring above). + keepalive = asyncio.create_task(_keepalive_progress(job_manager, MAP_PROGRESS_KEEPALIVE_SECONDS)) + try: + mapping_results = await asyncio.wait_for(loop.run_in_executor(ctx["pool"], blocking), timeout=map_budget) + except asyncio.TimeoutError as e: + raise TimeoutError( + f"Mapping for {score_set.urn} exceeded its {map_budget}s budget ({variant_count} variants); " + "the mapping service did not return in time." + ) from e + finally: + keepalive.cancel() + with contextlib.suppress(asyncio.CancelledError): + await keepalive logger.debug(msg="Done mapping variants.", extra=job_manager.logging_context()) job_manager.update_progress(80, 100, "Processing mapped variants.") @@ -130,7 +197,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Per-(target, alignment_level) QC records produced by the dcd-mapping API. # All records share the same tool_version because they come from a single run; - # we use that as the global ``mapping_api_version`` carried on each MappedVariant. + # we use that as the global ``mapping_api_version`` carried on each MappingRecord. target_mappings_payload = mapping_results.get("target_mappings") or [] tool_version = next( (tm.get("tool_version") for tm in target_mappings_payload if tm.get("tool_version")), @@ -173,12 +240,15 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.save_to_context({"mapped_hgnc_name": target_gene.mapped_hgnc_name}) logger.debug("Added mapped HGNC name to target gene.", extra=job_manager.logging_context()) + else: + target_gene.mapped_hgnc_name = None + logger.debug("No gene-level info found for target gene.", extra=job_manager.logging_context()) # add annotation layer info for annotation_layer in reference_metadata[target_gene_identifier]["layers"]: # ``annotation_layer`` arrives as a dcd-mapping wire code (``p``/``c``/``g``); # we persist metadata under the corresponding full-name enum value. - layer_name = AnnotationLayer.from_wire(annotation_layer).value + layer_name = SequenceLevel.from_wire(annotation_layer).value layer_premapped = reference_metadata[target_gene_identifier]["layers"][annotation_layer].get( "computed_reference_sequence" ) @@ -203,6 +273,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan target_gene.pre_mapped_metadata = cast(pre_mapped_metadata, JSONB) target_gene.post_mapped_metadata = cast(post_mapped_metadata, JSONB) + target_gene.uniprot_id_from_mapped_metadata = None + job_manager.db.add(target_gene) logger.debug("Added mapping metadata to target gene.", extra=job_manager.logging_context()) @@ -218,8 +290,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan continue target_gene_mapping = TargetGeneMapping( - target_gene_id=target_gene.id, - alignment_level=AnnotationLayer.from_wire(level_value), + target_gene=target_gene, + alignment_level=SequenceLevel.from_wire(level_value), preferred=bool(tm.get("preferred", False)), reference_assembly=tm.get("reference_assembly"), reference_accession=tm.get("reference_accession"), @@ -241,6 +313,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan alignment_metadata=tm.get("alignment_metadata"), vrs_version=tm.get("vrs_version"), mapped_date=mapping_results["mapped_date"], + job_run_id=job_manager.job_id, ) job_manager.db.add(target_gene_mapping) target_gene_mapping_by_key[(target_gene_identifier, level_value)] = target_gene_mapping @@ -255,12 +328,16 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan total_variants = len(mapped_scores) job_manager.save_to_context({"total_variants_to_process": total_variants}) - successful_mapped_variants = 0 + # Tally every record by its typed outcome; the mapped/failed/benign buckets are + # derived from this after the loop. Keeping all four keys preserves the + # intronic-vs-no-protein distinction in logs. + outcome_counts: Counter[MappingOutcome] = Counter() logger.info( f"Processing {total_variants} mapped variants for score set {score_set.urn}.", extra=job_manager.logging_context(), ) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id) + + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id, score_set_id=score_set.id) for mapped_score in mapped_scores: variant_urn = mapped_score.get("mavedb_id") variant = job_manager.db.scalars(select(Variant).where(Variant.urn == variant_urn)).one() @@ -268,23 +345,28 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.save_to_context({"processing_variant": variant.id}) logger.debug(f"Processing variant {variant.id}.", extra=job_manager.logging_context()) - # there should only be one current mapped variant per variant id, so update old mapped variant to current = false + # Only allow one live MappingRecord per variant. The prior live record (if any) is + # superseded by the new version below via supersede_with, which retires it (cascading to + # its allele links) and inserts the new record under one timestamp. existing_mapped_variant = ( - job_manager.db.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + job_manager.db.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - if existing_mapped_variant: job_manager.save_to_context({"existing_mapped_variant": existing_mapped_variant.id}) - existing_mapped_variant.current = False - job_manager.db.add(existing_mapped_variant) - logger.debug(msg="Set existing mapped variant to current = false.", extra=job_manager.logging_context()) - annotation_was_successful = mapped_score.get("pre_mapped") and mapped_score.get("post_mapped") - if annotation_was_successful: - successful_mapped_variants += 1 - job_manager.save_to_context({"successful_mapped_variants": successful_mapped_variants}) + # The typed outcome -- not allele presence -- decides success/benign/failure. + # Absent outcome means an older or malformed payload; fail fast. + raw_outcome = mapped_score.get("outcome") + if not raw_outcome: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} is missing its outcome." + ) + outcome = MappingOutcome(raw_outcome) + + outcome_counts[outcome] += 1 + job_manager.save_to_context({"outcome_counts": {o.value: n for o, n in outcome_counts.items()}}) # dcd-mapping guarantees both fields are set on every ScoreAnnotation, # including failed variants (the annotate step re-attributes failures to @@ -294,61 +376,134 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan score_alignment_level = mapped_score.get("alignment_level") if not score_target or not score_alignment_level: raise NonexistentMappingResultsError( - f"ScoreAnnotation for variant {variant_urn!r} is missing " - f"target_gene_identifier or alignment_level." + f"ScoreAnnotation for variant {variant_urn!r} is missing target_gene_identifier or alignment_level." ) + pre_mapped_allele: dict = mapped_score.get("pre_mapped") or {} + post_mapped_allele: dict = mapped_score.get("post_mapped") or {} + sequence_level = SequenceLevel.from_wire(score_alignment_level) + assay_level_hgvs = get_hgvs_from_post_mapped(post_mapped_allele, combine_cis=True) + + # dcd-mapping guarantees every mapped score is attributable to a TargetGeneMapping + # via (target_gene_identifier, alignment_level) -- including failed variants, which + # it re-attributes to the target's preferred layer. A miss implies a malformed payload. target_gene_mapping_row = target_gene_mapping_by_key.get((score_target, score_alignment_level)) - mapped_variant = MappedVariant( - pre_mapped=mapped_score.get("pre_mapped", null()), - post_mapped=mapped_score.get("post_mapped", null()), - hgvs_assay_level=get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), + if target_gene_mapping_row is None: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} has no TargetGeneMapping for " + f"(target={score_target!r}, alignment_level={score_alignment_level!r})." + ) + + mapping_record = MappingRecord( variant_id=variant.id, - modification_date=date.today(), + score_set_id=variant.score_set_id, + vrs_digest=pre_mapped_allele.get("id"), + pre_mapped=pre_mapped_allele or None, + assay_level=sequence_level, + hgvs_assay_level=assay_level_hgvs, mapped_date=mapping_results["mapped_date"], - vrs_version=mapped_score.get("vrs_version", null()), + vrs_version=mapped_score.get("vrs_version", None), mapping_api_version=tool_version, - error_message=mapped_score.get("error_message", null()), - target_gene_mapping_id=target_gene_mapping_row.id if target_gene_mapping_row else None, - alignment_level=AnnotationLayer.from_wire(score_alignment_level), + target_gene_mapping_id=target_gene_mapping_row.id, + alignment_level=sequence_level, at_mismatched_locus=mapped_score.get("at_mismatched_locus"), near_gap=mapped_score.get("near_gap"), - current=True, ) - - annotation_manager.add_annotation( - variant_id=variant.id, # type: ignore - annotation_type=AnnotationType.VRS_MAPPING, - version=tool_version, - status=AnnotationStatus.SUCCESS if annotation_was_successful else AnnotationStatus.FAILED, - failure_category=None - if annotation_was_successful - else AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, - annotation_data={ - "error_message": mapped_score.get("error_message", null()), - "annotation_metadata": { - "mapped_assay_level_hgvs": get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), - }, + if existing_mapped_variant: + # Retire the prior record (cascading to its allele links) and insert this one under a + # single timestamp, so the prior valid_to equals the new valid_from with no gap. + existing_mapped_variant.supersede_with(job_manager.db, mapping_record) + logger.debug( + msg="Superseded prior mapping record and its allele links.", extra=job_manager.logging_context() + ) + else: + job_manager.db.add(mapping_record) + + # The mapper emits a MappingRecord for EVERY variant, so "a record exists" carries no + # signal — the signal is whether a real allele resulted. MAPPED yields an authoritative + # allele -> present. A benign absence (intronic, synonymous) produces a record but no + # allele: an informative biological negative -> absent. FAILED -> failed. `reason` reuses + # the MappingOutcome vocabulary so the intronic-vs-no-protein distinction survives. + if outcome is MappingOutcome.MAPPED: + disposition = Disposition.PRESENT + elif outcome.is_benign_absence: + disposition = Disposition.ABSENT + else: + disposition = Disposition.FAILED + + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=disposition, + reason=outcome.value, + source_version=tool_version, + metadata={ + "mapped_assay_level_hgvs": assay_level_hgvs, + "error_message": mapped_score.get("error_message"), }, - current=True, ) - job_manager.db.add(mapped_variant) + # Only variants with a post-mapped representation yield an authoritative Allele; + # failed and benign-absent variants get a MappingRecord but no linked allele. + if post_mapped_allele: + # Recompute rather than trust the incoming id, which may predate normalization for a + # deletion/duplication. vrs_digest is the dedup key the whole allele graph hangs off. + canonical_post_mapped, allele_digest = canonical_variation_document( + post_mapped_allele, subject=f"variant {variant.urn}" + ) + # Verify that the post-mapped allele's sequence matches what SeqRepo holds for its accession. + if (seqrepo := getattr(ctx["seqrepo"], "sr", None)) is not None: + verify_allele_refget( + canonical_post_mapped, assay_level_hgvs or "", seqrepo, subject=f"variant {variant.urn}" + ) + + allele_draft = AlleleDbModel( + vrs_digest=allele_digest, + level=sequence_level, + hgvs_g=assay_level_hgvs if sequence_level == SequenceLevel.genomic else None, + hgvs_c=assay_level_hgvs if sequence_level == SequenceLevel.cdna else None, + hgvs_p=assay_level_hgvs if sequence_level == SequenceLevel.protein else None, + post_mapped=canonical_post_mapped, + ) + authoritative_allele = get_or_create_allele(job_manager.db, allele_draft) + job_manager.db.flush() + + # TODO#765: Mapping is not idempotent, so we must always create a new link. + job_manager.db.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=authoritative_allele.id, + is_authoritative=True, + ) + ) + logger.debug(msg="Linked mapped variant to authoritative allele.", extra=job_manager.logging_context()) + logger.debug(msg="Added new mapped variant to session.", extra=job_manager.logging_context()) + job_manager.db.flush() annotation_manager.flush() - if successful_mapped_variants == 0: + # Collapse the per-outcome tally into the three buckets the rest of the job reasons about. + mapped_count = outcome_counts[MappingOutcome.MAPPED] + failed_count = outcome_counts[MappingOutcome.FAILED] + skipped_count = sum(n for o, n in outcome_counts.items() if o.is_benign_absence) + + # State keys off genuine failures only: no failures -> complete (benign absences + # don't count); failures with nothing mapped -> failed; otherwise incomplete. + if failed_count == 0: + score_set.mapping_state = MappingState.complete + elif mapped_count == 0: score_set.mapping_state = MappingState.failed score_set.mapping_errors = {"error_message": "All variants failed to map."} - elif successful_mapped_variants < total_variants: - score_set.mapping_state = MappingState.incomplete else: - score_set.mapping_state = MappingState.complete + score_set.mapping_state = MappingState.incomplete job_manager.save_to_context( { - "successful_mapped_variants": successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "mapping_state": score_set.mapping_state.name, "mapping_errors": score_set.mapping_errors, "inserted_mapped_variants": len(mapped_scores), @@ -397,7 +552,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan logger.info(msg="Inserted mapped variants into db.", extra=job_manager.logging_context()) - if successful_mapped_variants == 0: + # Fail the job only on genuine failure with nothing mapped; all-benign is a success. + if mapped_count == 0 and failed_count > 0: logger.error(msg="No variants were successfully mapped.", extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -405,19 +561,27 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan data={ "score_set_id": score_set.id, "mapped_count": 0, - "unmapped_count": total_variants, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, }, failure_category=FailureCategory.VRS_MAPPING_FAILED, ) - logger.info(msg="Variant mapping job completed successfully.", extra=job_manager.logging_context()) + logger.info( + msg=( + f"Variant mapping job completed successfully: {mapped_count} mapped, " + f"{failed_count} failed, {skipped_count} skipped (intronic / no protein consequence)." + ), + extra=job_manager.logging_context(), + ) job_manager.db.flush() return JobExecutionOutcome.succeeded( data={ "score_set_id": score_set.id, - "mapped_count": successful_mapped_variants, - "unmapped_count": total_variants - successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, } ) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py new file mode 100644 index 000000000..2e1dc4a30 --- /dev/null +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -0,0 +1,797 @@ +"""Reverse translation worker job — builds cross-level HGVS equivalence classes. + +First refreshes every record's measured projection group (see :mod:`mavedb.worker.lib.projection`), the +measured variant described at each level. Then, for each mapped variant in the score set, calls +construct_equivalent_variants from the variant-annotation library to produce the coding/genomic HGVS +candidates encoding the same protein consequence, plus that protein consequence itself. Each candidate is +written as a non-authoritative Allele row linked to the existing MappingRecord via MappingRecordAllele. + +The library returns the equivalence class as a list of ProjectionPair objects, each a coding candidate and +its deterministic genomic projection — the same change at two levels. Each pair becomes a candidate +ProjectionGroup row, and both of its links point at it. A pair holding the measured allele is the measured +variant itself, so its other member joins the measured group instead; so does the protein consequence. +""" + +import asyncio +import dataclasses +import functools +import logging +from collections import Counter +from datetime import date +from typing import Any, Callable, NamedTuple, Sequence + +from ga4gh.vrs.extras.translator import AlleleTranslator +from sqlalchemy import func, select +from sqlalchemy.orm import Session +from variant_annotation import __version__ as variant_annotation_version +from variant_annotation.lib.accessions import looks_like_refseq_protein_accession +from variant_annotation.lib.translation import construct_equivalent_variants +from variant_annotation.lib.translation.types import ( + TranslationConfig, + TranslationErrorReason, + VariantInput, + WtCodonMode, +) + +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.hgvs import extract_accession, is_cis_phased_hgvs, strip_protein_prediction_parens +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele +from mavedb.lib.vrs_utils import translate_hgvs_to_variation, verify_allele_refget +from mavedb.models.allele import Allele as AlleleDbModel +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.enums.target_category import TargetCategory +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_accession import TargetAccession +from mavedb.models.target_gene import TargetGene +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.worker.jobs.utils.setup import validate_job_params +from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management +from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.projection import project_measured_variants +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator, uta_transcript_source + +logger = logging.getLogger(__name__) + +TranscriptResolver = Callable[[int | None, int, date | None, str | None], str | None] + +# Job defaults: full coding equivalence class — every synonymous codon, indels included. +# wt_codon_mode "all" requires include_indels=True. +_DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { + "include_indels": True, + "wt_codon_mode": WtCodonMode.ALL, + # The public UTA server refuses connections for minutes at a time; the library default (about 6 s + # in total) fails the whole batch on a blip. Waits 5, 10, 20, 40 and 80 s before giving up. + "upstream_max_attempts": 6, + "upstream_retry_backoff_seconds": 5.0, +} + +# A protein-coding target with no MANE Select transcript at its first loci (an intergenic tile, say) is +# given up on after this many lookups rather than queried once per record. +_MANE_ATTEMPTS_PER_TARGET = 5 + + +class _TranscriptResolution(NamedTuple): + """Pairs a mapping record with its pre-UTA transcript info: gene_transcript if the mapper + supplied a cdna row, protein_accession if NP_→NM_ resolution is still needed. Both None + means the record will be skipped.""" + + rec: MappingRecord + variant: Variant + gene_transcript: str | None + protein_accession: str | None + target_gene_id: int | None + + +def _classify_skip( + resolution: _TranscriptResolution, category: TargetCategory | None +) -> tuple[EventReason, Disposition]: + """Classify why a record was skipped, returning its reason and event disposition. + + ``no_assay_level_hgvs`` — no input to translate, we could not ask → ``not_applicable``. + ``transcript_unresolved`` — protein-coding target with no transcript, a recoverable pipeline + gap → ``failed``. ``no_coding_transcript`` — non-coding target has no protein consequence, a + biological negative → ``absent``. + """ + if not resolution.rec.hgvs_assay_level: + return (EventReason.NO_ASSAY_LEVEL_HGVS, Disposition.NOT_APPLICABLE) + if category == TargetCategory.protein_coding: + return (EventReason.TRANSCRIPT_UNRESOLVED, Disposition.FAILED) + return (EventReason.NO_CODING_TRANSCRIPT, Disposition.ABSENT) + + +def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: + """Resolve RefSeq protein accessions (NP_/XP_) to their preferred coding transcripts via UTA. + + Protein-level mappings carry no cdna transcript in TargetGeneMapping, so reverse + translation falls back to the NP_→NM_ association in UTA. Connection is opened and + closed per-call to avoid leaking connections across long-lived worker jobs. + """ + if not protein_accessions: + return {} + + with uta_transcript_source() as client: + return { + pro_ac: transcript + for pro_ac in sorted(protein_accessions) + if (transcript := client.transcript_for_protein(pro_ac)) is not None + } + + +def cdna_transcript_resolver( + db: Session, score_set_id: int, coordinates: WorkerCoordinateTranslator +) -> TranscriptResolver: + """Return a resolver from a record's ``(job_run_id, target_gene_id, mapped_date, genomic_hgvs)`` to its + cdna transcript. + + In order: + + 1. The run's own cdna mapping row. A record binds to it via ``job_run_id``; records with no + ``job_run_id`` (pre-column, or reshaped from legacy) use the day-granular + ``(target_gene_id, mapped_date)`` key, which cannot separate two runs on one calendar day. Within + either key, highest ``TargetGeneMapping.id`` wins. + 2. For records with no ``job_run_id``, the coding accession the legacy mapper recorded in the target's + ``post_mapped_metadata``: legacy mappings wrote no cdna row, and their genomic HGVS came from that + same run. + 3. For genomic records, the target's own accession when it is a transcript (NM_, ENST). + 4. For genomic records of a protein-coding target, the MANE Select transcript at the record's locus, + found once per target gene. Genomic-only targets (most saturation genome editing) have no other + transcript; this is the one the legacy ClinGen HGVS lookup reported for them. + + Protein records pass no ``genomic_hgvs``, so they stop after step 2 and keep their NP_→NM_ lookup. The + lookup tables are built once per score set here and closed over. A failed MANE Select lookup raises + rather than resolving to no transcript, which would erase a coding projection an earlier run wrote. + """ + by_job_run: dict[tuple[int, int], str | None] = {} + by_run_date: dict[tuple[int, date | None], str | None] = {} + for target_gene_id, mapped_date, job_run_id, reference_accession in ( + db.execute( + select( + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.job_run_id, + TargetGeneMapping.reference_accession, + ) + .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) + .where(TargetGene.score_set_id == score_set_id) + .where(TargetGeneMapping.alignment_level == SequenceLevel.cdna) + .where(TargetGeneMapping.reference_accession.isnot(None)) + .order_by(TargetGeneMapping.id) + ) + .tuples() + .all() + ): + if job_run_id is not None: + by_job_run[(job_run_id, target_gene_id)] = reference_accession + by_run_date[(target_gene_id, mapped_date)] = reference_accession + + legacy_by_target_gene: dict[int, str] = {} + accession_by_target_gene: dict[int, str] = {} + protein_coding: set[int] = set() + for target_gene_id, metadata, category, target_accession in ( + db.execute( + select(TargetGene.id, TargetGene.post_mapped_metadata, TargetGene.category, TargetAccession.accession) + .outerjoin(TargetAccession, TargetAccession.id == TargetGene.accession_id) + .where(TargetGene.score_set_id == score_set_id) + ) + .tuples() + .all() + ): + cdna = metadata.get("cdna") if isinstance(metadata, dict) else None + accessions = cdna.get("sequence_accessions") if isinstance(cdna, dict) else None + if accessions: + legacy_by_target_gene[target_gene_id] = accessions[0] + if target_accession and target_accession.startswith(("NM_", "ENST")): + accession_by_target_gene[target_gene_id] = target_accession + if category == TargetCategory.protein_coding: + protein_coding.add(target_gene_id) + + mane_by_target_gene: dict[int, str] = {} + mane_attempts: Counter[int] = Counter() + + def mane_select(target_gene_id: int, genomic_hgvs: str) -> str | None: + if target_gene_id in mane_by_target_gene: + return mane_by_target_gene[target_gene_id] + if target_gene_id not in protein_coding or mane_attempts[target_gene_id] >= _MANE_ATTEMPTS_PER_TARGET: + return None + transcript = coordinates.mane_select_transcript(genomic_hgvs) + mane_attempts[target_gene_id] += 1 + if transcript is not None: + mane_by_target_gene[target_gene_id] = transcript + return transcript + + def resolve( + job_run_id: int | None, target_gene_id: int, mapped_date: date | None, genomic_hgvs: str | None + ) -> str | None: + if job_run_id is not None: + transcript = by_job_run.get((job_run_id, target_gene_id)) + else: + transcript = by_run_date.get((target_gene_id, mapped_date)) or legacy_by_target_gene.get(target_gene_id) + if transcript is not None or genomic_hgvs is None: + return transcript + return accession_by_target_gene.get(target_gene_id) or mane_select(target_gene_id, genomic_hgvs) + + return resolve + + +def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationConfig: + """Build a TranslationConfig from optional job-param overrides merged with job defaults. + + Overrides win over _DEFAULT_TRANSLATION_CONFIG; wt_codon_mode accepts string values and + is coerced to the enum. Raises ValueError for unknown fields, invalid modes, or invalid + combinations (e.g. wt_codon_mode != "none" without include_indels). + """ + config_kwargs: dict[str, Any] = {**_DEFAULT_TRANSLATION_CONFIG, **(overrides or {})} + + allowed_fields = {field.name for field in dataclasses.fields(TranslationConfig)} + unknown_fields = set(config_kwargs) - allowed_fields + if unknown_fields: + raise ValueError( + f"Unknown translation_config option(s): {', '.join(sorted(unknown_fields))}. " + f"Valid options: {', '.join(sorted(allowed_fields))}." + ) + + raw_mode = config_kwargs.get("wt_codon_mode") + if raw_mode is not None: + try: + config_kwargs["wt_codon_mode"] = WtCodonMode(raw_mode) + except ValueError: + valid_modes = ", ".join(repr(mode.value) for mode in WtCodonMode) + raise ValueError( + f"Invalid translation_config wt_codon_mode {raw_mode!r}. Valid values: {valid_modes}." + ) from None + + try: + return TranslationConfig(**config_kwargs) + except ValueError as exc: + raise ValueError(f"Invalid translation_config: {exc}") from exc + + +def _annotate_translation( + annotation_manager: AnnotationStatusManager, + variant_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CROSS_LEVEL_TRANSLATION event for an allele (the translation is a variant-level fact). + + The single choke point for RT's status writes. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CROSS_LEVEL_TRANSLATION, + variant_id=variant_id, + disposition=disposition, + reason=reason, + source_version=variant_annotation_version, + metadata=meta or None, + ) + + +@with_pipeline_management +async def reverse_translate_variants_for_score_set( + ctx: dict, job_id: int, job_manager: JobManager +) -> JobExecutionOutcome: + """Build the cross-level HGVS equivalence class for every mapped variant in the score set. + + For each current MappingRecord with an hgvs_assay_level, collapses to a ProteinConsequence + and expands to all coding/genomic HGVS candidates via the variant-annotation library. + Each candidate is written as a non-authoritative Allele linked to the MappingRecord. + + job_params: score_set_id (int), correlation_id (str), + translation_config (dict, optional) — TranslationConfig overrides. + """ + job = job_manager.get_job() + validate_job_params(["score_set_id", "correlation_id"], job) + + score_set_id: int = job.job_params["score_set_id"] # type: ignore[index] + correlation_id: str = job.job_params["correlation_id"] # type: ignore[index] + translation_config = _build_translation_config(job.job_params.get("translation_config")) # type: ignore[union-attr] + score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == score_set_id)).one() + + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "reverse_translate_variants_for_score_set", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) + job_manager.update_progress(0, 100, "Starting reverse translation job.") + logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) + + coordinates = WorkerCoordinateTranslator(ctx["hdp"]) + resolve_cdna_transcript = cdna_transcript_resolver(job_manager.db, score_set_id, coordinates) + + # The measured groups come first: candidates that match the measured variant join its group below. + projection_counts = project_measured_variants( + job_manager.db, score_set_id, coordinates=coordinates, resolve_transcript=resolve_cdna_transcript + ) + logger.info( + msg=f"Measured projection groups: {dict(projection_counts)}.", + extra=job_manager.logging_context(), + ) + + # Load current authoritative MappingRecords with their Variant and TargetGeneMapping. The record's own + # TargetGeneMapping (its measured alignment) carries the run's job_run_id and mapped_date, which anchor + # the cdna transcript lookup below. + rows: Sequence[tuple[MappingRecord, Variant, int, date | None, int | None]] = ( + job_manager.db.execute( + select( + MappingRecord, + Variant, + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.job_run_id, + ) + .join(MappingRecordAllele, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .outerjoin(TargetGeneMapping, MappingRecord.target_gene_mapping_id == TargetGeneMapping.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + ) + .tuples() + .all() + ) + + annotation_counts: Counter[str] = Counter( + { + "translated": 0, + "failed": 0, + "skipped": 0, + "alleles_created": 0, + "fold_in_missed": 0, + } + ) + + if not rows: + logger.warning( + msg="No current and authoritative mapping records found for this score set.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) + + # Genomic/cdna records resolve via the cdna TargetGeneMapping; protein records have no + # cdna reference_accession, so collect their NP_ accessions for a batched NP_→NM_ UTA lookup. + transcript_resolutions: list[_TranscriptResolution] = [] + protein_accessions: set[str] = set() + for rec, variant, target_gene_id, mapped_date, record_job_run_id in rows: + genomic_hgvs = rec.hgvs_assay_level if rec.assay_level == SequenceLevel.genomic else None + coding_accession = resolve_cdna_transcript(record_job_run_id, target_gene_id, mapped_date, genomic_hgvs) + protein_accession = None + if not coding_accession and rec.hgvs_assay_level is not None: + raw_accession = extract_accession(rec.hgvs_assay_level) + if looks_like_refseq_protein_accession(raw_accession): + protein_accession = raw_accession + protein_accessions.add(raw_accession) + + transcript_resolutions.append( + _TranscriptResolution(rec, variant, coding_accession, protein_accession, target_gene_id) + ) + + transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) + + # Target category per gene — used to classify skips as recoverable (coding target, + # transcript unresolved) vs. correct (non-coding/regulatory, no protein consequence). + target_category_by_gene: dict[int, TargetCategory] = dict( + job_manager.db.execute( + select(TargetGene.id, TargetGene.category).where(TargetGene.score_set_id == score_set_id) + ) + .tuples() + .all() + ) + + # Build VariantInputs with the resolved coding transcript (p./c./g. all collapse to a + # ProteinConsequence on that transcript). Object identity is preserved so TranslationResult + # can be correlated back to its MappingRecord. Records with no transcript are skipped. + variant_inputs: list[Any] = [] + variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} + skipped_variants: list[_TranscriptResolution] = [] + cis_phased_variants: list[_TranscriptResolution] = [] + for p in transcript_resolutions: + # A cis-phased multivariant HGVS (NC_…:g.[a;b;…]) cannot be forward-translated: the engine's + # parser rejects the allele list at the opening "[". These are almost always true multi-mutant + # haplotypes, not single-codon components, so they have no meaningful cross-level sibling triple. + # Record a benign skip rather than send a guaranteed failure to the engine. + if p.rec.hgvs_assay_level and is_cis_phased_hgvs(p.rec.hgvs_assay_level): + cis_phased_variants.append(p) + continue + + transcript = p.gene_transcript or ( + transcript_by_protein.get(p.protein_accession) if p.protein_accession else None + ) + if not transcript or not p.rec.hgvs_assay_level: + skipped_variants.append(p) + continue + + inp = VariantInput(hgvs=p.rec.hgvs_assay_level, transcript=transcript) + variant_inputs.append(inp) + variant_input_map[id(inp)] = (p.rec, p.variant) + + total = len(variant_inputs) + job_manager.save_to_context( + { + "total_variants": total, + "skipped_variants": len(skipped_variants), + "cis_phased_skipped": len(cis_phased_variants), + } + ) + job_manager.update_progress( + 10, + 100, + f"Prepared {total} variants for reverse translation " + f"({len(skipped_variants)} no coding transcript, {len(cis_phased_variants)} cis-phased).", + ) + logger.info( + msg=( + f"Running reverse translation for {total} variants " + f"({len(skipped_variants)} skipped no-transcript, {len(cis_phased_variants)} skipped cis-phased)." + ), + extra=job_manager.logging_context(), + ) + + # construct_equivalent_variants is I/O-bound (blocks on a subprocess); run in the + # default thread pool rather than the process pool to avoid pickling the port objects. + loop = asyncio.get_running_loop() + job_manager.update_progress(20, 100, "Running reverse translation subprocess.") + # WtCodonMode.ALL reads the reference codon via TranscriptSource.codon_at, so pass a + # live UTA-backed source. codon_at is only touched in the post-subprocess WT-codon step, + # but the connection must outlive the executor call, so scope it around the whole call. + # (transcript_for_protein is redundant here -- the job already resolved every transcript + # and supplies it via VariantInput.transcript.) + with uta_transcript_source() as transcripts: + results, errors = await loop.run_in_executor( + None, + functools.partial( + construct_equivalent_variants, + variant_inputs, + transcripts=transcripts, + coordinates=coordinates, + config=translation_config, + ), + ) + + job_manager.update_progress(70, 100, "Writing translated alleles to database.") + logger.info( + msg=f"Translation complete: {len(results)} succeeded, {len(errors)} failed.", + extra=job_manager.logging_context(), + ) + + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set_id + ) + allele_translator = AlleleTranslator(ctx["seqrepo"]) + + current_record_ids = ( + select(MappingRecord.id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current.is_(True)) + ) + + # Derived links and candidate groups are deferred until every record is processed, so the prior derived + # set is superseded atomically: retire and insert share one timestamp. + new_links: list[MappingRecordAllele] = [] + new_groups: list[tuple[ProjectionGroup, list[MappingRecordAllele]]] = [] + + # The live authoritative link of each current record. It points at the record's measured projection + # group (written by project_measured_variants above), which this job adds alleles to but never rewrites. + authoritative_by_record: dict[int, MappingRecordAllele] = { + link.mapping_record_id: link + for link in job_manager.db.scalars( + select(MappingRecordAllele) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) + ).all() + } + seqrepo = getattr(ctx["seqrepo"], "sr", None) + + for result in results: + rec, variant = variant_input_map[id(result.input)] + authoritative = authoritative_by_record.get(rec.id) + measured_allele_id = authoritative.allele_id if authoritative is not None else None + measured_group = authoritative.projection_group if authoritative is not None else None + # Prediction parens (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay + # inputs, where the protein is already the authoritative allele. + hgvs_p = strip_protein_prediction_parens(result.hgvs_p) if result.hgvs_p else None + + candidate_count = 0 + linked_nucleotide = False + folded_in = False + # Equivalence generation may surface the same VRS object more than once; link each allele once. + seen_digests: set[str] = set() + failed_candidates: list[dict[str, str]] = [] + candidates_metadata: list[dict[str, Any]] = [] + + def translated(hgvs: str, level: SequenceLevel, hgvs_field: str) -> AlleleDbModel | None: + """The allele for one candidate HGVS, or None if it cannot be translated (intronic, malformed).""" + try: + variation = translate_hgvs_to_variation(hgvs, allele_translator) + except Exception as e: + logger.warning( + msg=f"Failed to translate candidate HGVS to VRS: {hgvs} ({e})", + extra=job_manager.logging_context(), + ) + failed_candidates.append({"hgvs": hgvs, "level": level.value, "error": str(e)}) + return None + + # Verify that the allele's sequence matches what SeqRepo holds for its accession. + if seqrepo is not None: + verify_allele_refget( + variation.model_dump(exclude_none=True), hgvs, seqrepo, subject=f"variant record {rec.id}" + ) + + # translate_hgvs_to_variation identifies every variation it returns. + assert variation.id is not None + allele = get_or_create_allele( + job_manager.db, + AlleleDbModel( + vrs_digest=variation.id, + # exclude_none mirrors the mapper's serialization. + post_mapped=variation.model_dump(exclude_none=True), + level=level, + **{hgvs_field: hgvs}, # type: ignore[arg-type] + ), + ) + job_manager.db.flush() + return allele + + def link(allele: AlleleDbModel) -> MappingRecordAllele: + seen_digests.add(allele.vrs_digest) + return MappingRecordAllele( + mapping_record_id=rec.id, score_set_id=rec.score_set_id, allele_id=allele.id, is_authoritative=False + ) + + # Each ProjectionPair is a coding candidate and its genomic projection: the same change at two levels. + # hgvs_g is None when the c→g projection failed, leaving a one-member group. Two spellings of one coding + # change (an insertion either side of an exon junction) identify to one allele, which links only once, + # so pairs with a genomic projection go first to keep that allele with its genomic partner. + for pair in sorted(result.projection_pairs, key=lambda pair: pair.hgvs_g is None): + members = [(pair.hgvs_c, SequenceLevel.cdna, "hgvs_c")] + if pair.hgvs_g is not None: + members.append((pair.hgvs_g, SequenceLevel.genomic, "hgvs_g")) + alleles = [ + allele for hgvs, level, field in members if (allele := translated(hgvs, level, field)) is not None + ] + # Unseen alleles, once each: a pair's two members can identify to one VRS object. + fresh = list({a.vrs_digest: a for a in alleles if a.vrs_digest not in seen_digests}.values()) + + # A pair holding the measured allele is the measured variant itself: its other member joins the + # measured group rather than forming a candidate group. + is_measured = measured_allele_id is not None and any(a.id == measured_allele_id for a in alleles) + candidates_metadata.append( + { + "hgvs_c": pair.hgvs_c, + "hgvs_g": pair.hgvs_g, + "variant_type": pair.variant_type, + "measured": is_measured, + } + ) + if is_measured: + folded_in = True + for allele in fresh: + if allele.id != measured_allele_id: + partner = link(allele) + partner.projection_group = measured_group + new_links.append(partner) + candidate_count += 1 + continue + + if fresh: + group = ProjectionGroup( + mapping_record_id=rec.id, + score_set_id=rec.score_set_id, + hgvs_c=pair.hgvs_c, + hgvs_g=pair.hgvs_g, + hgvs_p=hgvs_p, + ) + new_groups.append((group, [link(allele) for allele in fresh])) + candidate_count += len(fresh) + linked_nucleotide = True + + # The protein consequence is the measured variant's own, so it joins the measured group. + if hgvs_p is not None: + protein = translated(hgvs_p, SequenceLevel.protein, "hgvs_p") + if protein is not None and protein.id != measured_allele_id and protein.vrs_digest not in seen_digests: + apex = link(protein) + apex.projection_group = measured_group + new_links.append(apex) + candidate_count += 1 + + # A nucleotide measured allele that RT linked candidates to but never matched: its candidates are distinct + # encodings of the consequence. Expected for changes RT's codon-level candidates can't express; otherwise + # the mapper and RT produced different VRS objects for one change. + fold_in_missed = ( + linked_nucleotide and not folded_in and authoritative is not None and rec.assay_level in NUCLEOTIDE_LEVELS + ) + annotation_counts["fold_in_missed"] += fold_in_missed + + annotation_counts["alleles_created"] += candidate_count + annotation_metadata = { + "hgvs_input": result.input.hgvs, + "candidates": candidates_metadata, + "hgvs_p": result.hgvs_p, + "alleles_created": candidate_count, + "failed_candidates": failed_candidates, + "fold_in_missed": fold_in_missed, + } + + # No translatable candidates and failures mean the variant failed reverse translation. No + # failures and no candidates is a success with no alleles created. + if candidate_count == 0 and failed_candidates: + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.FAILED, + reason=EventReason.TRANSLATION_FAILED, + error_message="All candidate HGVS failed VRS translation.", + metadata=annotation_metadata, + ) + else: + annotation_counts["translated"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.TRANSLATED, + metadata=annotation_metadata, + ) + + # A record the library couldn't ask about because UTA was unavailable keeps its derived links and + # candidate groups: its inputs are unchanged since the run that produced them, so they are still that + # run's answer. Every other outcome, including a skip or a failed translation, is this run's answer and + # replaces the prior set. + unavailable_record_ids = { + variant_input_map[id(error.input)][0].id + for error in errors + if error.reason is TranslationErrorReason.UPSTREAM_UNAVAILABLE + } + measured_group_ids = ( + select(MappingRecordAllele.projection_group) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.projection_group.isnot(None)) + .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) + ) + + # Supersede the prior candidate groups and derived links under one timestamp. Measured groups belong to + # project_measured_variants and are never retired here; their derived links are, and are re-added above. + # TODO#765: re-runs retire and recreate the whole derived set because re-mapping re-mints + # records; idempotent records would allow unchanged links to stay live. + superseded_at = job_manager.db.scalar(select(func.now())) + ProjectionGroup.supersede_live_where( + job_manager.db, + [group for group, _ in new_groups], + ProjectionGroup.id.not_in(measured_group_ids), + ProjectionGroup.mapping_record_id.in_(current_record_ids), + ProjectionGroup.mapping_record_id.not_in(unavailable_record_ids), + at=superseded_at, + ) + job_manager.db.flush() + for group, links in new_groups: + for group_link in links: + group_link.projection_group = group.id + new_links.extend(links) + + MappingRecordAllele.supersede_live_where( + job_manager.db, + new_links, + MappingRecordAllele.is_authoritative.is_(False), + MappingRecordAllele.mapping_record_id.in_(current_record_ids), + MappingRecordAllele.mapping_record_id.not_in(unavailable_record_ids), + at=superseded_at, + ) + + # The library types each error's reason: NOT_TRANSLATABLE is a benign structural gap (the + # protein consequence's edit type — ins/delins/dup/fs/ext/stop-loss — has no DNA equivalence + # class to construct), UPSTREAM_UNAVAILABLE is a UTA outage that outlasted the library's + # retries, and FAILED is a genuine translation error. Split on that typed reason rather than + # pattern-matching the engine's error text. + for error in errors: + _rec, variant = variant_input_map[id(error.input)] + if error.reason is TranslationErrorReason.NOT_TRANSLATABLE: + # A structural gap ("we could not ask"), not a failure — count it as a skip and record + # NOT_APPLICABLE so it never pollutes the failed tally (mirrors NO_ASSAY_LEVEL_HGVS). + annotation_counts["skipped"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.NOT_APPLICABLE, + reason=EventReason.NOT_TRANSLATABLE, + metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, + ) + continue + + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, + variant_id=variant.id, + disposition=Disposition.FAILED, + # An upstream outage says nothing about the variant, so keep it out of translation_error. + reason=( + EventReason.API_ERROR + if error.reason is TranslationErrorReason.UPSTREAM_UNAVAILABLE + else EventReason.TRANSLATION_ERROR + ), + metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, + ) + + # skipped already holds the NOT_TRANSLATABLE skips counted above; add the transcript-unresolved + # skips (skipped_variants) rather than overwriting. + annotation_counts["skipped"] += len(skipped_variants) + for p in skipped_variants: + category = target_category_by_gene.get(p.target_gene_id) if p.target_gene_id is not None else None + reason, disposition = _classify_skip(p, category) + _annotate_translation( + annotation_manager, + variant_id=p.variant.id, + disposition=disposition, + reason=reason, + metadata={"hgvs_input": p.rec.hgvs_assay_level}, + ) + + # add cis-phased multivariant inputs as NOT_APPLICABLE skips + annotation_counts["skipped"] += len(cis_phased_variants) + for p in cis_phased_variants: + _annotate_translation( + annotation_manager, + variant_id=p.variant.id, + disposition=Disposition.NOT_APPLICABLE, + reason=EventReason.CIS_PHASED_UNSUPPORTED, + metadata={"hgvs_input": p.rec.hgvs_assay_level}, + ) + + annotation_manager.flush() + + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) + logger.info( + msg=( + f"Reverse translation complete: {annotation_counts['translated']} translated, " + f"{annotation_counts['failed']} failed, {annotation_counts['skipped']} skipped, " + f"{annotation_counts['alleles_created']} alleles created." + ), + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + + if annotation_counts["fold_in_missed"]: + logger.warning( + msg=( + f"{annotation_counts['fold_in_missed']} measured alleles matched none of their reverse translation " + "candidates; their events carry fold_in_missed in metadata." + ), + extra=job_manager.logging_context(), + ) + + if annotation_counts["translated"] == 0 and annotation_counts["failed"] > 0: + logger.error( + msg="All variant reverse translations failed.", + extra=job_manager.logging_context(), + ) + return JobExecutionOutcome.failed( + reason="All variant reverse translations failed.", + data=outcome_data, + failure_category=FailureCategory.DATA_ERROR, + ) + + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/src/mavedb/worker/jobs_overview.md b/src/mavedb/worker/jobs_overview.md index bfca70c6a..2f83e6154 100644 --- a/src/mavedb/worker/jobs_overview.md +++ b/src/mavedb/worker/jobs_overview.md @@ -28,8 +28,10 @@ Router (score_sets.py) │ • start_pipeline JobRun (entrypoint) │ • create_variants_for_score_set JobRun │ • map_variants_for_score_set JobRun (depends on create_variants) - │ • submit_to_car JobRun (depends on map_variants) - │ • link_gnomad JobRun (depends on submit_to_car) + │ • reverse_translate_variants_for_score_set JobRun (depends on map_variants) + │ • submit_to_car JobRun (depends on reverse_translate) + │ • warm_clingen_cache JobRun (depends on submit_to_car) + │ • link_gnomad JobRun (depends on warm_clingen_cache) │ • ... more annotation jobs with dependencies │ • JobDependency records linking them │ @@ -58,7 +60,7 @@ ARQ Worker picks up start_pipeline │ ├─ 6. ARQ picks up map_variants_for_score_set │ ├─ @with_pipeline_management runs job, marks SUCCEEDED - │ └─ coordinate_pipeline() finds submit_to_car, submit_uniprot, etc. → enqueue + │ └─ coordinate_pipeline() finds reverse_translate, submit_uniprot → enqueue │ ├─ 7... Continues until all jobs complete │ @@ -84,7 +86,7 @@ ARQ Cron Scheduler (or manual enqueue) └─ 3. No pipeline coordination (job has no pipeline_id) ``` -Example: `cleanup_stalled_jobs` runs every 30 minutes via ARQ cron to find and handle stuck jobs. +Example: `cleanup_stalled_jobs` runs every 10 minutes via ARQ cron to find and handle stuck jobs. ## Key Models @@ -104,6 +106,7 @@ The central record for every job execution: | `max_retries` | Maximum retry attempts (default: 3) | | `retry_count` | Current retry attempt count | | `progress_current/total/message` | Progress tracking fields | +| `progress_updated_at` | Heartbeat bumped on every committing progress update; `cleanup_stalled_jobs` reaps RUNNING jobs whose heartbeat goes stale | | `error_message/error_traceback` | Error details on failure | | `failure_category` | `FailureCategory` enum for retry classification | | `metadata_` | JSONB for retry history, result snapshots, etc. | diff --git a/src/mavedb/worker/lib/decorators/job_management.py b/src/mavedb/worker/lib/decorators/job_management.py index 58aad427c..0dcb463ad 100644 --- a/src/mavedb/worker/lib/decorators/job_management.py +++ b/src/mavedb/worker/lib/decorators/job_management.py @@ -5,6 +5,7 @@ Includes JobManager injection for advanced operations and robust error handling. """ +import asyncio import functools import inspect import logging @@ -15,7 +16,7 @@ from mavedb.lib.slack import send_slack_error, send_slack_job_error, send_slack_job_failure from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.worker.lib.decorators.utils import ensure_ctx, ensure_job_id, ensure_session_ctx, is_test_mode from mavedb.worker.lib.managers import JobManager from mavedb.worker.lib.managers.constants import TERMINAL_JOB_STATUSES @@ -143,6 +144,43 @@ async def _execute_managed_job(func: Callable[..., Awaitable[JobExecutionOutcome return result + # The coroutine is being cancelled — either ARQ hit job_timeout, or cleanup_stalled_jobs + # aborted this attempt (Job.abort()) as part of stall recovery. CancelledError is a + # BaseException, so the `except Exception` handler below never sees it; without this branch + # the JobRun row would be orphaned as RUNNING forever while the coroutine dies. + # + # We mark the row terminally (FAILED / TIMEOUT) so DB state matches reality, then RE-RAISE. + # Re-raising is essential: it lets ARQ record the cancellation as the job's result, which is + # what makes a concurrent Job.abort() confirm the job actually died. + except asyncio.CancelledError: + logger.warning(f"Job {job_id} cancelled (timeout or stall-recovery abort); marking FAILED before re-raising") + try: + db_session.rollback() + if job_manager is not None: + job_manager.fail_job( + result=JobExecutionOutcome.failed( + reason="Job cancelled due to timeout, stall-recovery, or internal ARQ abort", + data={"reason": "cancelled"}, + failure_category=FailureCategory.TIMEOUT, + ) + ) + db_session.commit() + job = job_manager.get_job() + send_slack_job_failure( + job_urn=job.urn, + job_function=job.job_function, + reason="Job cancelled due to timeout.", + failure_category=str(FailureCategory.TIMEOUT), + retry_count=job.retry_count, + max_retries=job.max_retries, + will_retry=False, + ) + + except Exception as inner_e: + logger.critical(f"Failed to mark cancelled job {job_id} as failed: {inner_e}") + send_slack_error(inner_e) + raise + except Exception as e: # Prioritize salvaging lifecycle state will_retry = False diff --git a/src/mavedb/worker/lib/managers/pipeline_manager.py b/src/mavedb/worker/lib/managers/pipeline_manager.py index 68f56c9d4..8c0f6c418 100644 --- a/src/mavedb/worker/lib/managers/pipeline_manager.py +++ b/src/mavedb/worker/lib/managers/pipeline_manager.py @@ -64,6 +64,7 @@ ) from mavedb.worker.lib.managers.utils import ( arq_job_id, + arq_queue_name, construct_bulk_cancellation_result, job_dependency_is_met, job_should_be_skipped_due_to_unfulfillable_dependency, @@ -1278,7 +1279,11 @@ async def _enqueue_in_arq(self, job: JobRun, is_retry: bool) -> None: try: defer_by = timedelta(seconds=job.retry_delay_seconds if is_retry and job.retry_delay_seconds else 0) arq_success = await self.redis.enqueue_job( - job.job_function, job.id, _defer_by=defer_by, _job_id=arq_job_id(job) + job.job_function, + job.id, + _defer_by=defer_by, + _job_id=arq_job_id(job), + _queue_name=arq_queue_name(job), ) except Exception as e: logger.debug(f"ARQ enqueue operation failed for job {job.urn}: {e}") diff --git a/src/mavedb/worker/lib/managers/utils.py b/src/mavedb/worker/lib/managers/utils.py index d6a266647..49135599c 100644 --- a/src/mavedb/worker/lib/managers/utils.py +++ b/src/mavedb/worker/lib/managers/utils.py @@ -12,6 +12,7 @@ import redis.exceptions import requests.exceptions import sqlalchemy.exc +from arq.constants import default_queue_name from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import DependencyType, FailureCategory, JobStatus @@ -44,6 +45,16 @@ OSError: FailureCategory.NETWORK_ERROR, } +FAST_LANE_QUEUE_NAME = "arq:queue:fast-lane" +"""ARQ queue consumed by a dedicated worker process, independent of the main worker's job slots.""" + +FAST_LANE_JOB_FUNCTIONS = frozenset({"start_pipeline", "create_variants_for_score_set"}) +"""Short jobs that must not wait behind hour-long mapping jobs holding every main-worker slot. + +``start_pipeline`` is included because it enqueues ``create_variants_for_score_set``; routing only the +latter would leave a new pipeline stuck behind mapping before variant creation is ever enqueued. +""" + def classify_exception(exc: Exception) -> FailureCategory: """Map an exception to a FailureCategory. Uses isinstance to match parent classes.""" @@ -65,6 +76,15 @@ def arq_job_id(job: JobRun) -> str: return f"{job.urn}#{job.retry_count or 0}" +def arq_queue_name(job: JobRun) -> str: + """Return the ARQ queue a JobRun is enqueued on. + + Every ARQ call that takes a queue name (``enqueue_job``, ``Job.status``, ``Job.abort``) must resolve it + here; a status lookup against the wrong queue reports a queued job as ``not_found``. + """ + return FAST_LANE_QUEUE_NAME if job.job_function in FAST_LANE_JOB_FUNCTIONS else default_queue_name + + def construct_bulk_cancellation_result(reason: str) -> JobExecutionOutcome: """Construct a standardized JobExecutionOutcome for bulk job cancellations. diff --git a/src/mavedb/worker/lib/projection.py b/src/mavedb/worker/lib/projection.py new file mode 100644 index 000000000..a6395f3d9 --- /dev/null +++ b/src/mavedb/worker/lib/projection.py @@ -0,0 +1,276 @@ +"""Measured projection groups: each mapping record's measured variant described at every level it has. + +A record's measured group holds its measured variant's HGVS at the measured level and its projections to +the other levels: the coding HGVS of a genomic measurement (on the record's transcript) or the genomic HGVS +of a coding one, plus the protein consequence (``c_to_p``). They are computed from +the mapping alone, so every nucleotide record gets them, including variants reverse translation cannot +reach: intronic and UTR changes, multi-codon indels, start and stop changes. + +The group carries strings only. Alleles join it through their links: the authoritative link here, the rest +from reverse translation when it constructs the same change (see the reverse translation job). +""" + +import logging +import re +from collections import Counter +from dataclasses import dataclass +from datetime import date +from typing import Callable, Optional, cast + +from sqlalchemy import CursorResult, select, true +from sqlalchemy.orm import Session +from variant_annotation.lib.translation import coding_to_genomic, genomic_to_coding + +from mavedb.lib.hgvs import extract_accession, is_cis_phased_hgvs, strip_protein_prediction_parens +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + +logger = logging.getLogger(__name__) + +TranscriptResolver = Callable[[Optional[int], int, Optional[date], Optional[str]], Optional[str]] + +_SYNONYMOUS_STOP = re.compile(r"p\.Ter(\d+)Ter$") +_SPANLESS_IDENTITY = re.compile(r"^(?P[^:]+):g\.(?P\d+)(?P[ACGTN]{2,})=$") + + +@dataclass(frozen=True) +class MeasuredHgvs: + """A measured variant's HGVS at each level; ``None`` where a level has none or could not be projected.""" + + hgvs_c: Optional[str] = None + hgvs_g: Optional[str] = None + hgvs_p: Optional[str] = None + + +def _expand_reference_identity(hgvs: str) -> str: + """Give a multi-base reference identity the span its reference implies. + + Legacy mappings wrote a codon-level identity as ``g.100CCT=``: one position with a three-base + reference, which hgvs rejects. ``g.100_102=`` is the same variant in valid HGVS. + """ + match = _SPANLESS_IDENTITY.match(hgvs) + if match is None: + return hgvs + start = int(match["start"]) + return f"{match['accession']}:g.{start}_{start + len(match['ref']) - 1}=" + + +def _protein_consequence(coordinates: WorkerCoordinateTranslator, hgvs_c: str) -> Optional[str]: + """The protein consequence of a coding variant, or ``None`` when it is unknown (``p.?``: intronic, UTR).""" + hgvs_p = strip_protein_prediction_parens(coordinates.c_to_p(hgvs_c)) + if hgvs_p.endswith(":p.?"): + return None + + # hgvs writes a synonymous stop as ``p.Ter778Ter``; HGVS writes every synonymous change with ``=``. + return _SYNONYMOUS_STOP.sub(r"p.Ter\1=", hgvs_p) + + +def project_measured_hgvs( + assay_level: SequenceLevel, + hgvs: str, + *, + transcript: Optional[str], + coordinates: WorkerCoordinateTranslator, +) -> tuple[MeasuredHgvs, list[str]]: + """Describe a measured variant at every level, returning the description and any projection errors. + + Each projection is attempted independently, so a failed protein consequence still keeps the coding + projection. A genomic measurement is normalized before it is projected, and the group carries the + normalized form; one with no coding transcript has only its genomic HGVS. Coding and genomic levels are + projected by variant-annotation's :func:`coding_to_genomic` and :func:`genomic_to_coding`, so a change no + single change describes at the other level (a codon split by an intron) has no projection there. + """ + errors: list[str] = [] + if assay_level == SequenceLevel.protein: + return MeasuredHgvs(hgvs_p=hgvs), errors + + hgvs_c: Optional[str] = None + hgvs_g: Optional[str] = None + if assay_level == SequenceLevel.genomic: + hgvs_g = _expand_reference_identity(hgvs) + # A reference identity has nothing to trim, and hgvs would collapse its span to one position. + if not hgvs_g.endswith("="): + try: + hgvs_g = coordinates.normalize(hgvs) + except Exception as e: + # A variant hgvs cannot normalize (a reference mismatch, say) still projects as written. + logger.debug("Could not normalize %s: %s", hgvs, e) + + if transcript is not None: + try: + hgvs_c = genomic_to_coding(hgvs_g, transcript, coordinates) + except Exception as e: + errors.append(f"g_to_c: {e}") + + else: + hgvs_c = hgvs + try: + hgvs_g = coding_to_genomic(hgvs, coordinates) + except Exception as e: + errors.append(f"c_to_g: {e}") + + hgvs_p: Optional[str] = None + if hgvs_c is not None: + try: + hgvs_p = _protein_consequence(coordinates, hgvs_c) + except Exception as e: + errors.append(f"c_to_p: {e}") + + return MeasuredHgvs(hgvs_c=hgvs_c, hgvs_g=hgvs_g, hgvs_p=hgvs_p), errors + + +def project_measured_variants( + db: Session, + score_set_id: int, + *, + coordinates: WorkerCoordinateTranslator, + resolve_transcript: TranscriptResolver, + missing_only: bool = False, +) -> Counter[str]: + """Write or refresh the measured projection group of every current mapping record in a score set. + + ``missing_only`` restricts the run to records with no measured group yet, skipping the projection of + records that have one. + + A record whose live measured group already holds the same description is left alone, so a re-run + writes nothing. Otherwise its group is superseded and its authoritative link is superseded by a copy + pointing at the new group. A record whose projection raises keeps the group it has, so an unreachable + data provider cannot erase earlier projections. + + Superseding a group retires the other links in it, because they described the old group: the measured + variant's partner at the other nucleotide level and its protein consequence, both added by reverse + translation. Until reverse translation runs again on the score set, the record serves neither allele + nor the ClinVar, gnomAD and VEP annotations reached through them. Candidate groups are not touched. + The reverse translation job runs this first, so it re-attaches them in the same run; a standalone run + reports the retired links as ``detached_rt_links`` and must be followed by reverse translation for + each score set with a non-zero count. ``missing_only`` runs never supersede a group. + + Does not commit. + """ + counts: Counter[str] = Counter() + # Scoped through variants: mapping_records has no score-set index, so filtering on its own score_set_id + # would scan the table once per score set. + rows = ( + db.execute( + select( + MappingRecord, + MappingRecordAllele, + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.job_run_id, + ) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .outerjoin(TargetGeneMapping, TargetGeneMapping.id == MappingRecord.target_gene_mapping_id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.projection_group.is_(None) if missing_only else true()) + ) + .tuples() + .all() + ) + + group_ids = [link.projection_group for _, link, *_ in rows if link.projection_group is not None] + live_groups = { + group.id: group + for group in db.scalars( + select(ProjectionGroup).where(ProjectionGroup.id.in_(group_ids)).where(ProjectionGroup.current) + ) + } + + pending: list[tuple[MappingRecord, MappingRecordAllele, MeasuredHgvs]] = [] + for record, link, target_gene_id, mapped_date, job_run_id in rows: + hgvs = record.hgvs_assay_level + if not hgvs or is_cis_phased_hgvs(hgvs): + counts["skipped_no_hgvs" if not hgvs else "skipped_cis_phased"] += 1 + continue + + level = SequenceLevel(record.assay_level) + transcript = None + resolve_errors: list[str] = [] + if level == SequenceLevel.genomic and target_gene_id is not None: + try: + transcript = resolve_transcript(job_run_id, target_gene_id, mapped_date, hgvs) + except Exception as e: + resolve_errors.append(f"transcript: {e}") + elif level == SequenceLevel.cdna: + transcript = extract_accession(hgvs) + + measured, errors = project_measured_hgvs(level, hgvs, transcript=transcript, coordinates=coordinates) + errors = resolve_errors + errors + current = live_groups.get(link.projection_group) if link.projection_group is not None else None + if errors: + counts["projection_errors"] += 1 + logger.debug("Projection errors for mapping record %s (%s): %s", record.id, hgvs, errors) + # An error may be transient (the data provider unreachable), so it never replaces a group a + # previous run wrote; a record with no group yet still gets the levels that did project. + if current is not None: + counts["kept_on_error"] += 1 + continue + + if current is not None and (current.hgvs_c, current.hgvs_g, current.hgvs_p) == ( + measured.hgvs_c, + measured.hgvs_g, + measured.hgvs_p, + ): + counts["unchanged"] += 1 + continue + + pending.append((record, link, measured)) + + if not pending: + return counts + + # Retire before inserting: the live-unique indexes on links (one per record and allele, one authoritative + # per record) must see the old rows closed. Both statements and the inserts below take the transaction's + # now(), so the handoff is gap-free. + prior_group_ids = [link.projection_group for _, link, _ in pending if link.projection_group is not None] + if prior_group_ids: + detached = cast( + CursorResult, + db.execute( + MappingRecordAllele.retire_live_where( + MappingRecordAllele.projection_group.in_(prior_group_ids), + MappingRecordAllele.is_authoritative.is_(False), + ) + ), + ) + if detached.rowcount: + counts["detached_rt_links"] += detached.rowcount + db.execute(ProjectionGroup.retire_live_where(ProjectionGroup.id.in_(prior_group_ids))) + db.execute(MappingRecordAllele.retire_live_where(MappingRecordAllele.id.in_([link.id for _, link, _ in pending]))) + + groups = [ + ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=measured.hgvs_c, + hgvs_g=measured.hgvs_g, + hgvs_p=measured.hgvs_p, + ) + for record, _, measured in pending + ] + db.add_all(groups) + db.flush() + + db.add_all( + MappingRecordAllele( + mapping_record_id=link.mapping_record_id, + score_set_id=link.score_set_id, + allele_id=link.allele_id, + is_authoritative=True, + projection_group=group.id, + ) + for (_, link, _), group in zip(pending, groups) + ) + db.flush() + + counts["projected"] += len(pending) + return counts diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py new file mode 100644 index 000000000..dd2e19d61 --- /dev/null +++ b/src/mavedb/worker/lib/translation_ports.py @@ -0,0 +1,159 @@ +"""Worker-side adapters for the variant_annotation translation ports. + +construct_equivalent_variants depends on two protocols — CoordinateTranslator +and TranscriptSource. This module supplies the worker's CoordinateTranslator +(WorkerCoordinateTranslator) and a factory (uta_transcript_source) for the +TranscriptSource, which is variant_annotation's own UTA-backed UtaClient. + +WorkerCoordinateTranslator defers AssemblyMapper initialization until the first +call — the hgvs library makes network calls on mapper construction that must not +fire in unit-test contexts where construct_equivalent_variants is mocked. +""" + +from __future__ import annotations + +import contextlib +import os +from collections.abc import Generator +from typing import Any + +from variant_annotation.lib.clients.uta import UtaClient + +# The public UTA server refuses connections for minutes at a time; the client default (about 3 s in +# total) fails the job on a blip. Waits 2, 4, 8, 16 and 32 s before giving up. +UTA_MAX_ATTEMPTS = 6 +UTA_BACKOFF_SECONDS = 2.0 + +_COMPLEMENT = str.maketrans("ACGTN", "TGCAN") + + +@contextlib.contextmanager +def uta_transcript_source() -> Generator[UtaClient]: + """Yield a UTA-backed TranscriptSource over a connection scoped to the block. + + Backs both the NP_→NM_ association lookup and the WT-codon read used by + WtCodonMode.ALL (TranscriptSource.codon_at). The client reconnects when UTA drops + the connection and closes it on exit, so callers must use the client within the + ``with`` block. + """ + uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() + if not uta_db_url: + raise RuntimeError("UTA_DB_URL must be set to resolve transcript facts (NP_→NM_ associations and WT codons).") + with UtaClient.from_url(uta_db_url, max_attempts=UTA_MAX_ATTEMPTS, backoff_seconds=UTA_BACKOFF_SECONDS) as client: + yield client + + +def _reverse_complement(sequence: str) -> str: + return sequence.translate(_COMPLEMENT)[::-1] + + +class WorkerCoordinateTranslator: + """CoordinateTranslator backed by the worker's HGVS data provider.""" + + def __init__(self, hdp: Any) -> None: + self._hdp = hdp + self._parser: Any = None + self._mapper: Any = None + self._literal_mapper: Any = None + self._normalizer: Any = None + + def _ensure_initialized(self) -> None: + if self._mapper is None: + import hgvs.assemblymapper + import hgvs.normalizer # type: ignore[import-untyped] + import hgvs.parser + + self._parser = hgvs.parser.Parser() + self._normalizer = hgvs.normalizer.Normalizer(self._hdp) + self._mapper = hgvs.assemblymapper.AssemblyMapper( + self._hdp, + assembly_name="GRCh38", + alt_aln_method="splign", + ) + # Unnormalized, so each end of a change maps where it is and a delins stays a delins rather than + # becoming ``inv`` (see c_to_g_literal). + self._literal_mapper = hgvs.assemblymapper.AssemblyMapper( + self._hdp, + assembly_name="GRCh38", + alt_aln_method="splign", + normalize=False, + ) + + def normalize(self, g_hgvs: str) -> str: + """Trim and 3'-shift a genomic variant on its reference sequence, keeping it representable in VRS. + + A genomic delins that shares a base with its insertion, at an exon boundary, otherwise projects to + an intron-spanning coding delins with no protein consequence instead of the coding deletion it is. + hgvs rewrites a reverse-complement delins as ``inv``, which VRS cannot represent, so that is + written back as the equivalent delins. + """ + self._ensure_initialized() + variant = self._normalizer.normalize(self._parser.parse(g_hgvs)) + if variant.posedit.edit.type == "inv": + import hgvs.edit # type: ignore[import-untyped] + + start, end = variant.posedit.pos.start.base, variant.posedit.pos.end.base + ref = self._hdp.get_seq(variant.ac, start - 1, end).upper() + variant.posedit.edit = hgvs.edit.NARefAlt(ref=ref, alt=_reverse_complement(ref)) + + return str(variant) + + def c_to_p(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_p(self._parser.parse(c_hgvs))) + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + self._ensure_initialized() + return str(self._mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) + + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + """Project to coding without normalization; variant-annotation uses it to check a projection's span and + to rewrite an ``inv`` as delins.""" + self._ensure_initialized() + return str(self._literal_mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) + + def c_to_g(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) + + def mane_select_transcript(self, g_hgvs: str) -> str | None: + """The MANE Select RefSeq transcript overlapping a genomic variant, or ``None`` if there is none. + + Reads cdot's per-build ``tag``, so it needs a cdot data provider. Where MANE Select transcripts of two + genes overlap the variant, the lowest accession wins, so the choice is stable across runs. HGVS that + does not parse has no locus and gives ``None``. Data provider failures raise, so callers can tell an + outage from a locus with no MANE transcript; this relies on a provider that raises on one, as + :func:`~mavedb.data_providers.services.cdot_rest` does, since cdot's own REST client reports an outage + as no transcripts. + """ + self._ensure_initialized() + try: + variant = self._parser.parse(g_hgvs) + except Exception: + return None + + candidates = sorted( + tx["tx_ac"] + for tx in self._hdp.get_tx_for_region( + variant.ac, "splign", variant.posedit.pos.start.base - 1, variant.posedit.pos.end.base + ) + if tx["tx_ac"].startswith("NM_") + ) + for accession in candidates: + transcript = self._hdp._get_transcript(accession) or {} + build = transcript.get("genome_builds", {}).get("GRCh38", {}) + if "MANE Select" in (build.get("tag") or ""): + return accession + + return None + + def c_to_g_literal(self, c_hgvs: str) -> str: + """Project to genomic without normalization; variant-annotation uses it to check a projection's span + and to rewrite an ``inv`` as delins.""" + self._ensure_initialized() + return str(self._literal_mapper.c_to_g(self._parser.parse(c_hgvs))) + + def c_to_n(self, c_hgvs: str) -> str: + """Project to transcript (``n.``) coordinates, which count exonic bases only.""" + self._ensure_initialized() + return str(self._literal_mapper.c_to_n(self._parser.parse(c_hgvs))) diff --git a/src/mavedb/worker/pipeline_management.md b/src/mavedb/worker/pipeline_management.md index 12354746d..9064fdb74 100644 --- a/src/mavedb/worker/pipeline_management.md +++ b/src/mavedb/worker/pipeline_management.md @@ -242,13 +242,14 @@ When a job fails: ### Stalled Job Recovery -The `cleanup_stalled_jobs` cron job (runs every 30 minutes) catches jobs stuck in intermediate states: - -| State | Timeout | Action | -|-------|---------|--------| -| `QUEUED` | 10 minutes | Fail → retry if eligible | -| `RUNNING` | 60 minutes | Fail → retry if eligible | -| `PENDING` (in pipeline) | 30 minutes | Fail → retry if eligible | +The `cleanup_stalled_jobs` cron job (runs every 10 minutes) catches jobs stuck in intermediate states: + +| State | Stall signal | Action | +|-------|--------------|--------| +| `QUEUED` | Never started and absent from ARQ's Redis queue | Fail → retry if eligible | +| `RUNNING` | `progress_updated_at` idle > 30 minutes (`PROGRESS_STALL_MINUTES`), or running > 23.5 hours (`RUNNING_TIMEOUT_MINUTES`) | Abort the ARQ attempt; once it is confirmed dead, fail → retry if eligible. An abort that cannot be confirmed leaves the job in place. | +| `PENDING` (in pipeline) | Created > 5 minutes ago and runnable | Fail → retry if eligible | +| Pipeline (non-terminal, no active jobs) | Created > 5 minutes ago | Re-coordinate to settle the pipeline status | ## See Also diff --git a/src/mavedb/worker/settings/__init__.py b/src/mavedb/worker/settings/__init__.py index af2e6a275..9f0c43198 100644 --- a/src/mavedb/worker/settings/__init__.py +++ b/src/mavedb/worker/settings/__init__.py @@ -4,16 +4,17 @@ - constants: Environment variable configuration - redis: Redis connection settings - lifecycle: Worker startup/shutdown hooks -- worker: Main ARQ worker configuration class +- worker: Main and fast-lane ARQ worker configuration classes The settings are designed to be modular and easily testable, with clear separation between infrastructure and application concerns. """ from .redis import RedisWorkerSettings -from .worker import ArqWorkerSettings +from .worker import ArqFastLaneWorkerSettings, ArqWorkerSettings __all__ = [ + "ArqFastLaneWorkerSettings", "ArqWorkerSettings", "RedisWorkerSettings", ] diff --git a/src/mavedb/worker/settings/lifecycle.py b/src/mavedb/worker/settings/lifecycle.py index 54a0b4c76..5be66bd2d 100644 --- a/src/mavedb/worker/settings/lifecycle.py +++ b/src/mavedb/worker/settings/lifecycle.py @@ -9,7 +9,7 @@ from concurrent import futures -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo_data_proxy def standalone_ctx(): @@ -18,6 +18,7 @@ def standalone_ctx(): ctx["pool"] = futures.ProcessPoolExecutor() ctx["redis"] = None # Redis connection can be set up here if needed. ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} # Additional context setup can be added here as needed. @@ -37,6 +38,7 @@ async def shutdown(ctx): async def on_job_start(ctx): ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index 6a558e6b6..645809c20 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -7,11 +7,13 @@ - Lifecycle hooks for startup/shutdown and job execution - Timeout and logging configuration -This is the main configuration class used to start the ARQ worker. +ArqWorkerSettings configures the main worker; ArqFastLaneWorkerSettings configures the +fast-lane worker that runs short jobs independently of the main worker's slots. """ from mavedb.lib.logging.canonical import log_job from mavedb.worker.jobs import BACKGROUND_CRONJOBS, BACKGROUND_FUNCTIONS +from mavedb.worker.lib.managers.utils import FAST_LANE_JOB_FUNCTIONS, FAST_LANE_QUEUE_NAME from mavedb.worker.settings.lifecycle import on_job_end, on_job_start, shutdown, startup from mavedb.worker.settings.redis import RedisWorkerSettings @@ -25,7 +27,13 @@ # driver, enabling incremental migration of job functions without touching # the FastAPI layer. Once all jobs use async sessions, raise MAX_JOBS to 10+. MAX_JOBS = 2 -JOB_TIMEOUT_SECONDS = 3 * 60 * 60 # 3 hours — matches RUNNING_TIMEOUT_MINUTES (150 min) with buffer +# ARQ's hard coroutine-kill ceiling. Kept deliberately high (24h) so ARQ is the *last* resort: +# our own DB-driven sweeper (cleanup_stalled_jobs) should detect and recover stalls long before +# this fires, keeping the JobRun state machine in sync with reality. VEP fan-out over large allele +# sets can legitimately run for hours, so a low ceiling would kill healthy work. +# cleanup's RUNNING_TIMEOUT_MINUTES backstop sits ~30 min under this to try and sweep these jobs before +# ARQ's hard timeout fires. +JOB_TIMEOUT_SECONDS = 24 * 60 * 60 # 24 hours class ArqWorkerSettings: @@ -44,3 +52,31 @@ class ArqWorkerSettings: max_jobs = MAX_JOBS job_timeout = JOB_TIMEOUT_SECONDS + # Required for cleanup_stalled_jobs to abort a wedged RUNNING job's coroutine (via Job.abort()) + # before re-driving it. Without this, ARQ ignores abort requests and the sweeper cannot safely + # confirm a stalled job is dead before retrying it. + allow_abort_jobs = True + + +class ArqFastLaneWorkerSettings: + """ + Settings for the fast-lane ARQ worker, run as a separate process from the main worker. + + Consumes only FAST_LANE_QUEUE_NAME so variant creation never waits for a main-worker slot. The main + worker still registers these functions to drain any already on the default queue. ARQ reads settings + from the class ``__dict__``, so attributes are not inherited from ArqWorkerSettings. No cron jobs: + those run once, on the main worker. + """ + + on_startup = startup + on_shutdown = shutdown + on_job_start = on_job_start + on_job_end = on_job_end + after_job_end = log_job + redis_settings = RedisWorkerSettings + queue_name = FAST_LANE_QUEUE_NAME + functions: list = [f for f in BACKGROUND_FUNCTIONS if f.__name__ in FAST_LANE_JOB_FUNCTIONS] + + max_jobs = MAX_JOBS + job_timeout = JOB_TIMEOUT_SECONDS + allow_abort_jobs = True diff --git a/tests/conftest.py b/tests/conftest.py index 34e366392..d2825fcdb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,11 +95,11 @@ def session(postgresql): Base.metadata.create_all(bind=engine) # Create a unique index for the published_variants_materialized_view to - # enforce uniqueness on (variant_id, mapped_variant_id, score_set_id). This + # enforce uniqueness on (variant_id, mapping_record_id, score_set_id). This # allows us to test mat view refreshes that require this constraint. session.execute( text("""CREATE UNIQUE INDEX IF NOT EXISTS published_variants_mv_unique_idx - ON published_variants_materialized_view (variant_id, mapped_variant_id, score_set_id)"""), + ON published_variants_materialized_view (variant_id, mapping_record_id, score_set_id)"""), ) session.commit() diff --git a/tests/conftest_optional.py b/tests/conftest_optional.py index 7e3f28b2f..d61b07c52 100644 --- a/tests/conftest_optional.py +++ b/tests/conftest_optional.py @@ -4,7 +4,7 @@ from concurrent import futures from inspect import getsourcefile from posixpath import abspath -from unittest.mock import patch +from unittest.mock import Mock, patch import cdot.hgvs.dataproviders import pytest @@ -12,6 +12,7 @@ from arq.worker import Worker from biocommons.seqrepo import SeqRepo from fastapi.testclient import TestClient +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from httpx import AsyncClient from sqlalchemy import Column, Float, Integer, MetaData, String, Table @@ -106,6 +107,7 @@ async def on_startup(ctx): async def on_job(ctx): ctx["db"] = session ctx["hdp"] = data_provider + ctx["seqrepo"] = Mock(spec=SeqRepoDataProxy) # mirrors lifecycle.on_job_start ctx["state"] = {} ctx["pool"] = futures.ProcessPoolExecutor() @@ -131,6 +133,7 @@ async def on_job(ctx): def standalone_worker_context(data_provider, arq_redis): yield { "hdp": data_provider, + "seqrepo": Mock(spec=SeqRepoDataProxy), # mirrors lifecycle.on_job_start "state": {}, "job_id": "test_job", "redis": arq_redis, diff --git a/mypy_stubs/ga4gh/va_spec/__init__.pyi b/tests/data_providers/__init__.py similarity index 100% rename from mypy_stubs/ga4gh/va_spec/__init__.pyi rename to tests/data_providers/__init__.py diff --git a/tests/data_providers/test_services.py b/tests/data_providers/test_services.py new file mode 100644 index 000000000..ba1e96221 --- /dev/null +++ b/tests/data_providers/test_services.py @@ -0,0 +1,93 @@ +# ruff: noqa: E402 +from unittest.mock import patch + +import pytest +import requests +import requests_mock + +pytest.importorskip("boto3") +pytest.importorskip("cdot") +pytest.importorskip("hgvs") +pytest.importorskip("variant_annotation") + +from mavedb.data_providers.services import CDOT_REQUEST_TIMEOUT, StrictRESTDataProvider, cdot_rest +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator + +CDOT_URL = "http://cdot.test" +TRANSCRIPT = "NM_007294.4" +TRANSCRIPT_URL = f"{CDOT_URL}/transcript/{TRANSCRIPT}" +GENOMIC_HGVS = "NC_000017.11:g.43045712T>C" +REGION_URL = f"{CDOT_URL}/transcripts/region/NC_000017.11/splign/43045711/43045712" +MANE_TRANSCRIPT_DATA = {"id": TRANSCRIPT, "genome_builds": {"GRCh38": {"tag": "MANE Select"}}} + + +@pytest.fixture +def provider() -> StrictRESTDataProvider: + return StrictRESTDataProvider(url=CDOT_URL) + + +@pytest.mark.unit +def test_cdot_rest_builds_the_strict_provider(): + # The genome FASTA files behind the default seqfetcher exist only in the container. + with patch("mavedb.data_providers.services.seqfetcher", return_value=None): + assert isinstance(cdot_rest(), StrictRESTDataProvider) + + +@pytest.mark.unit +def test_a_missing_transcript_is_none_and_cached(provider): + with requests_mock.Mocker() as m: + m.get(TRANSCRIPT_URL, status_code=404) + + assert provider._get_transcript(TRANSCRIPT) is None + assert provider._get_transcript(TRANSCRIPT) is None + assert m.call_count == 1 + + +@pytest.mark.unit +def test_an_outage_raises_and_is_not_cached(provider): + with requests_mock.Mocker() as m: + m.get(TRANSCRIPT_URL, status_code=503) + with pytest.raises(requests.HTTPError): + provider._get_transcript(TRANSCRIPT) + + m.get(TRANSCRIPT_URL, json=MANE_TRANSCRIPT_DATA) + assert provider._get_transcript(TRANSCRIPT) == MANE_TRANSCRIPT_DATA + + +@pytest.mark.unit +def test_requests_carry_a_timeout(provider): + with requests_mock.Mocker() as m: + m.get(TRANSCRIPT_URL, json=MANE_TRANSCRIPT_DATA) + provider._get_transcript(TRANSCRIPT) + + assert m.last_request.timeout == CDOT_REQUEST_TIMEOUT + + +@pytest.mark.unit +def test_mane_select_lookup_raises_during_an_outage(provider): + coordinates = WorkerCoordinateTranslator(provider) + with requests_mock.Mocker() as m: + m.get(REGION_URL, status_code=503) + + with pytest.raises(requests.HTTPError): + coordinates.mane_select_transcript(GENOMIC_HGVS) + + +@pytest.mark.unit +def test_mane_select_lookup_finds_the_tagged_transcript(provider): + coordinates = WorkerCoordinateTranslator(provider) + with requests_mock.Mocker() as m: + m.get(REGION_URL, json={"results": [{"tx_ac": TRANSCRIPT}]}) + m.get(TRANSCRIPT_URL, json=MANE_TRANSCRIPT_DATA) + + assert coordinates.mane_select_transcript(GENOMIC_HGVS) == TRANSCRIPT + + +@pytest.mark.unit +def test_mane_select_lookup_skips_a_listed_transcript_cdot_no_longer_has(provider): + coordinates = WorkerCoordinateTranslator(provider) + with requests_mock.Mocker() as m: + m.get(REGION_URL, json={"results": [{"tx_ac": TRANSCRIPT}]}) + m.get(TRANSCRIPT_URL, status_code=404) + + assert coordinates.mane_select_transcript(GENOMIC_HGVS) is None diff --git a/tests/db/__init__.py b/tests/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/db/test_mixins.py b/tests/db/test_mixins.py new file mode 100644 index 000000000..cd7ef373d --- /dev/null +++ b/tests/db/test_mixins.py @@ -0,0 +1,334 @@ +"""Unit tests for the ValidTime mixin. + +Tested against minimal, purpose-built models rather than any real MaveDB models. ``_VtParent`` is +a cascade-bearing entity (natural key ``key``); ``_VtChild`` is a leaf link (natural key +``(parent_id, tag)``). Both carry a partial unique index over their natural key +``WHERE valid_to IS NULL``, mirroring the contract real consumers must honor. + +Many checks pass an explicit ``at`` that differs from the DB clock: a single-transaction test cannot +otherwise distinguish a deliberate shared timestamp from the coincidental same-transaction handoff +the gap bug relied on, so the explicit ``at`` is what actually proves the timestamp is threaded. +""" + +# ruff: noqa: E402 +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import Column, ForeignKey, Index, Integer, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import aliased, declarative_base, relationship + +from mavedb.db.mixins import ValidTime + +VtBase = declarative_base() + +# Fixed, deterministic windows — far from the test's transaction clock so an accidental +# server_default (func.now()) stamp would be visibly wrong rather than coincidentally equal. +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +class _VtParent(ValidTime, VtBase): + __tablename__ = "vt_parents" + + id = Column(Integer, primary_key=True) + key = Column(Integer, nullable=False) + children = relationship("_VtChild", back_populates="parent") + + __retire_cascade__ = ("children",) + __table_args__ = (Index("uq_vt_parents_live", "key", unique=True, postgresql_where=text("valid_to IS NULL")),) + + +class _VtChild(ValidTime, VtBase): + __tablename__ = "vt_children" + + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey("vt_parents.id"), nullable=False) + tag = Column(Integer, nullable=False) + parent = relationship("_VtParent", back_populates="children") + + __table_args__ = ( + Index("uq_vt_children_live", "parent_id", "tag", unique=True, postgresql_where=text("valid_to IS NULL")), + ) + + +@pytest.fixture(autouse=True) +def vt_tables(session): + """Create the throwaway ValidTime tables on the test engine, drop them after. Autouse so every + test in the module gets the tables without naming the fixture in its signature.""" + engine = session.get_bind() + VtBase.metadata.create_all(bind=engine) + yield + # Release any locks the test's still-open transaction holds before dropping: a post-commit + # attribute access reopens a transaction holding ACCESS SHARE on these tables, and DROP TABLE + # would block on it forever (this fixture tears down while the session fixture is still open). + session.rollback() + VtBase.metadata.drop_all(bind=engine) + + +def _parent(session, key, *, valid_from=None): + p = _VtParent(key=key, valid_from=valid_from) if valid_from else _VtParent(key=key) + session.add(p) + session.commit() + return p + + +def _child(session, parent, tag, *, valid_from=None): + c = ( + _VtChild(parent_id=parent.id, tag=tag, valid_from=valid_from) + if valid_from + else _VtChild(parent_id=parent.id, tag=tag) + ) + session.add(c) + session.commit() + return c + + +class TestCurrentAndAsOf: + def test_fresh_row_is_current(self, session): + p = _parent(session, 1) + assert p.valid_to is None + assert p.current is True + + def test_retired_row_is_not_current(self, session): + p = _parent(session, 1) + p.retire(session, at=T1) + assert p.current is False + + def test_current_expression_filters_to_live_rows(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + session.commit() + + rows = session.scalars(select(_VtParent).where(_VtParent.current)).all() + assert [r.id for r in rows] == [live.id] + + def test_as_of_selects_rows_live_at_timestamp(self, session): + # Window [T0, T2): explicit valid_from and a retire at T2. + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def live_ids(ts): + return [r.id for r in session.scalars(select(_VtParent).where(_VtParent.as_of(ts))).all()] + + assert live_ids(T0 - timedelta(days=1)) == [] # before the window opens + assert live_ids(T0) == [p.id] # at valid_from (inclusive) + assert live_ids(T1) == [p.id] # inside the window + assert live_ids(T2) == [] # at valid_to (exclusive) + assert live_ids(T2 + timedelta(days=1)) == [] # after the window closes + + +class TestLiveAt: + """``live_at(as_of)`` is the one-call ``as_of(ts) if ts else current`` — same result, but it also + works on an ``aliased()`` entity, which the raw ternary at call sites does too, but only clumsily.""" + + def test_live_at_none_is_current_on_instance(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + assert live.live_at(None) is True + assert retired.live_at(None) is False + + def test_live_at_ts_matches_the_window_on_instance(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + assert p.live_at(T0 - timedelta(days=1)) is False # before the window opens + assert p.live_at(T0) is True # at valid_from (inclusive) + assert p.live_at(T1) is True # inside the window + assert p.live_at(T2) is False # at valid_to (exclusive) + + def test_live_at_expression_agrees_with_current_and_as_of(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def ids(pred): + return [r.id for r in session.scalars(select(_VtParent).where(pred)).all()] + + assert ids(_VtParent.live_at(None)) == ids(_VtParent.current) == [] # retired -> not current + assert ids(_VtParent.live_at(T1)) == ids(_VtParent.as_of(T1)) == [p.id] + + def test_live_at_is_alias_aware(self, session): + # The point of live_at: it scopes to the alias, not the base table (a plain classmethod on an + # aliased entity is the trap this avoids). Both the compiled SQL and a query through the alias. + p = _parent(session, 1, valid_from=T0) + session.commit() + alias = aliased(_VtParent, name="pa") + assert "pa." in str(alias.live_at(T1)) + rows = session.scalars(select(alias).where(alias.live_at(T1))).all() + assert [r.id for r in rows] == [p.id] + + +class TestRetire: + def test_retire_closes_valid_to(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + assert c.valid_to == T1 + + def test_retire_is_idempotent(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + c.retire(at=T2) # already closed — keeps the original valid_to + assert c.valid_to == T1 + + def test_leaf_retire_needs_no_session(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire() # no cascade declared, so no session required + session.commit() + assert c.valid_to is not None + + def test_retire_cascades_to_live_child_links(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + p.retire(session, at=T1) + session.commit() + + assert p.valid_to == T1 + assert c1.valid_to == T1 # cascade closes the children at the same instant + assert c2.valid_to == T1 + + def test_cascade_retire_without_session_raises(self, session): + p = _parent(session, 1) + with pytest.raises(ValueError, match="cascade-retire"): + p.retire() # cascade declared but no session to issue the child UPDATE + + def test_cascade_runs_even_when_row_already_closed(self, session): + # A half-finished prior retire (parent closed, child not) must still converge. + p = _parent(session, 1) + c = _child(session, p, 1) + p.valid_to = T0 # simulate parent closed without its child + session.commit() + + p.retire(session, at=T1) # idempotent on the parent, but still cascades + session.commit() + assert p.valid_to == T0 # unchanged (idempotent) + assert c.valid_to == T1 # child still retired + + +class TestRetireLiveWhere: + def test_retires_matching_live_rows(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert c1.valid_to == T1 + assert c2.valid_to == T1 + + def test_leaves_already_retired_rows_untouched(self, session): + p = _parent(session, 1) + already = _child(session, p, 1) + already.retire(at=T0) + session.commit() + live = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert already.valid_to == T0 # retired row keeps its original valid_to + assert live.valid_to == T1 + + +class TestSupersedeWith: + def test_gap_free_handoff_with_explicit_at(self, session): + old = _parent(session, 1, valid_from=T0) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + # The retired valid_to equals the successor's valid_from exactly — no point-in-time hole. + assert old.valid_to == T1 + assert new.valid_from == T1 + + def test_default_at_handoff_is_gap_free(self, session): + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new) # at defaults to a single captured func.now() + session.commit() + + assert old.valid_to is not None + assert old.valid_to == new.valid_from + + def test_cascades_child_links_at_the_handoff_timestamp(self, session): + old = _parent(session, 1) + child = _child(session, old, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + assert old.valid_to == T1 + assert new.valid_from == T1 + assert child.valid_to == T1 # the old parent's child link retired under the same timestamp + + def test_respects_partial_unique_index_via_flush_ordering(self, session): + # old and new share natural key=1; supersede flushes the retire before inserting the + # successor, so the two are never simultaneously live and the unique index does not trip. + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() # would raise IntegrityError if both were live at the INSERT + + live = session.scalars(select(_VtParent).where(_VtParent.key == 1, _VtParent.current)).all() + assert [r.id for r in live] == [new.id] + + +class TestSupersedeLiveWhere: + def test_stamps_explicit_valid_from_on_inserts(self, session): + # Without flushing, valid_from would be None under the server_default path; supersede sets it + # explicitly, which is what keeps the handoff gap-free across a later-transaction flush. + new = _VtChild(parent_id=1, tag=1) + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == 1, at=T1) + assert new.valid_from == T1 + + def test_gap_free_handoff_with_reused_natural_key(self, session): + p = _parent(session, 1) + old = _child(session, p, 1) + new = _VtChild(parent_id=p.id, tag=1) # same natural key (parent_id, tag) + + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == p.id, at=T1) + session.commit() # reused key: retire executes before the insert, so the index holds + + assert old.valid_to == T1 + assert new.valid_from == T1 + live = session.scalars(select(_VtChild).where(_VtChild.parent_id == p.id, _VtChild.current)).all() + assert [r.id for r in live] == [new.id] + + def test_empty_new_rows_retires_the_scope(self, session): + # A re-run that produces nothing should still withdraw the prior live set. + p = _parent(session, 1) + old = _child(session, p, 1) + + _VtChild.supersede_live_where(session, [], _VtChild.parent_id == p.id, at=T1) + session.commit() + + assert old.valid_to == T1 + assert session.scalars(select(_VtChild).where(_VtChild.current)).all() == [] + + def test_refuses_a_cascade_bearing_class(self, session): + # Bulk supersede cannot fire __retire_cascade__, so it must refuse rather than orphan links. + with pytest.raises(ValueError, match="__retire_cascade__"): + _VtParent.supersede_live_where(session, [], _VtParent.key == 1) + + +class TestPartialUniqueIndexBackstop: + def test_two_live_rows_for_one_key_raise(self, session): + # The DB index is the loud backstop: forgetting to retire a predecessor before inserting its + # successor fails at flush instead of silently leaving two live rows. + session.add(_VtParent(key=1)) + session.add(_VtParent(key=1)) + with pytest.raises(IntegrityError): + session.commit() diff --git a/tests/db/test_timeouts.py b/tests/db/test_timeouts.py new file mode 100644 index 000000000..8b5e5d911 --- /dev/null +++ b/tests/db/test_timeouts.py @@ -0,0 +1,121 @@ +# ruff: noqa: E402 + +import logging + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import create_engine, text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session +from sqlalchemy.pool import NullPool +from starlette_context import context, request_cycle_context + +from mavedb.db import timeouts +from mavedb.db.timeouts import ( + CSV_STATEMENT_TIMEOUT_SECONDS, + allow_long_statements, + apply_api_timeouts, + observe_statements, +) + + +def _show(connection_or_session, setting: str) -> str: + return connection_or_session.execute(text(f"SHOW {setting}")).scalar() + + +@pytest.fixture +def engine(session): + return session.get_bind() + + +@pytest.mark.unit +class TestAllowLongStatements: + def test_raises_statement_timeout_for_the_transaction(self, session): + allow_long_statements(session) + + assert _show(session, "statement_timeout") == f"{CSV_STATEMENT_TIMEOUT_SECONDS}s" + + +@pytest.mark.unit +class TestApplyApiTimeouts: + @pytest.fixture + def api_engine(self, engine): + api_engine = create_engine(engine.url, poolclass=NullPool) + apply_api_timeouts(api_engine) + apply_api_timeouts(api_engine) # idempotent + return api_engine + + def test_new_connections_carry_the_api_limits(self, api_engine): + with api_engine.connect() as conn: + assert _show(conn, "statement_timeout") == "30s" + assert _show(conn, "lock_timeout") == "5s" + assert _show(conn, "idle_in_transaction_session_timeout") == "1min" + + def test_csv_override_raises_and_then_restores_the_limit(self, api_engine): + with Session(api_engine) as session: + allow_long_statements(session) + assert _show(session, "statement_timeout") == f"{CSV_STATEMENT_TIMEOUT_SECONDS}s" + session.rollback() + assert _show(session, "statement_timeout") == "30s" + + def test_zero_disables_the_statement_timeout_everywhere(self, engine, monkeypatch): + monkeypatch.setattr(timeouts, "API_STATEMENT_TIMEOUT_SECONDS", 0) + disabled_engine = create_engine(engine.url, poolclass=NullPool) + apply_api_timeouts(disabled_engine) + + with Session(disabled_engine) as session: + assert _show(session, "statement_timeout") == "0" + allow_long_statements(session) + assert _show(session, "statement_timeout") == "0" + + def test_other_engines_are_unbounded(self, engine): + with engine.connect() as conn: + assert _show(conn, "statement_timeout") == "0" + + +@pytest.mark.unit +class TestObserveStatements: + def test_logs_a_slow_statement_without_its_parameters(self, session, engine, caplog): + observe_statements(engine, threshold_seconds=0.05) + + with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): + session.execute(text("SELECT pg_sleep(0.1), :secret"), {"secret": "hunter2"}) + + messages = [record.message for record in caplog.records] + assert any("Slow database statement" in message and "pg_sleep" in message for message in messages) + assert not any("hunter2" in message for message in messages) + + def test_does_not_log_a_fast_statement(self, session, engine, caplog): + observe_statements(engine, threshold_seconds=5) + + with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): + session.execute(text("SELECT 1")) + + assert not caplog.records + + def test_counts_each_statement_in_the_request_context(self, session, engine): + observe_statements(engine, threshold_seconds=5) + + with request_cycle_context({}): + session.execute(text("SELECT 1")) + session.execute(text("SELECT 2")) + assert context["db_statement_count"] == 2 + + def test_counting_outside_a_request_is_a_no_op(self, session, engine): + observe_statements(engine, threshold_seconds=5) + + session.execute(text("SELECT 1")) + + assert not context.exists() + + def test_logs_a_statement_cancelled_by_the_timeout(self, session, engine, caplog): + observe_statements(engine, threshold_seconds=0.05) + session.execute(text("SET LOCAL statement_timeout = '100ms'")) + + with caplog.at_level(logging.WARNING, logger="mavedb.db.timeouts"): + with pytest.raises(OperationalError): + session.execute(text("SELECT pg_sleep(1)")) + + assert any("Slow database statement" in record.message for record in caplog.records) diff --git a/tests/fixtures/canonical_accession_sequences.json b/tests/fixtures/canonical_accession_sequences.json new file mode 100644 index 000000000..b2166a015 --- /dev/null +++ b/tests/fixtures/canonical_accession_sequences.json @@ -0,0 +1,21 @@ +{ + "_comment": "Canonical accession sequences (NCBI records) and their GA4GH refgets, used to assert that every writer of VRS alleles resolves an accession to the same refget. expected_refget is sha512t24u(sequence). non_canonical_refgets are values that have been minted in error and must never be produced. Keep identical to the copy in the API repo (tests/fixtures/canonical_accession_sequences.json).", + "accessions": { + "NM_007294.3": { + "sequence": "GTACCTTGATTTCGTATTCTGAGAGGCTGCTGCTTAGCGGTAGCCCCTTGGTTTCCGTGGCAACGGAAAAGCGCGGGAATTACAGATAAATTAAAACTGCGACTGCGCGGCGTGAGCTCGCTGAGACTTCCTGGACGGGGGACAGGCTGTGGGGTTTCTCAGATAACTGGGCCCCTGCGCTCAGGAGGCCTTCACCCTCTGCTCTGGGTAAAGTTCATTGGAACAGAAAGAAATGGATTTATCTGCTCTTCGCGTTGAAGAAGTACAAAATGTCATTAATGCTATGCAGAAAATCTTAGAGTGTCCCATCTGTCTGGAGTTGATCAAGGAACCTGTCTCCACAAAGTGTGACCACATATTTTGCAAATTTTGCATGCTGAAACTTCTCAACCAGAAGAAAGGGCCTTCACAGTGTCCTTTATGTAAGAATGATATAACCAAAAGGAGCCTACAAGAAAGTACGAGATTTAGTCAACTTGTTGAAGAGCTATTGAAAATCATTTGTGCTTTTCAGCTTGACACAGGTTTGGAGTATGCAAACAGCTATAATTTTGCAAAAAAGGAAAATAACTCTCCTGAACATCTAAAAGATGAAGTTTCTATCATCCAAAGTATGGGCTACAGAAACCGTGCCAAAAGACTTCTACAGAGTGAACCCGAAAATCCTTCCTTGCAGGAAACCAGTCTCAGTGTCCAACTCTCTAACCTTGGAACTGTGAGAACTCTGAGGACAAAGCAGCGGATACAACCTCAAAAGACGTCTGTCTACATTGAATTGGGATCTGATTCTTCTGAAGATACCGTTAATAAGGCAACTTATTGCAGTGTGGGAGATCAAGAATTGTTACAAATCACCCCTCAAGGAACCAGGGATGAAATCAGTTTGGATTCTGCAAAAAAGGCTGCTTGTGAATTTTCTGAGACGGATGTAACAAATACTGAACATCATCAACCCAGTAATAATGATTTGAACACCACTGAGAAGCGTGCAGCTGAGAGGCATCCAGAAAAGTATCAGGGTAGTTCTGTTTCAAACTTGCATGTGGAGCCATGTGGCACAAATACTCATGCCAGCTCATTACAGCATGAGAACAGCAGTTTATTACTCACTAAAGACAGAATGAATGTAGAAAAGGCTGAATTCTGTAATAAAAGCAAACAGCCTGGCTTAGCAAGGAGCCAACATAACAGATGGGCTGGAAGTAAGGAAACATGTAATGATAGGCGGACTCCCAGCACAGAAAAAAAGGTAGATCTGAATGCTGATCCCCTGTGTGAGAGAAAAGAATGGAATAAGCAGAAACTGCCATGCTCAGAGAATCCTAGAGATACTGAAGATGTTCCTTGGATAACACTAAATAGCAGCATTCAGAAAGTTAATGAGTGGTTTTCCAGAAGTGATGAACTGTTAGGTTCTGATGACTCACATGATGGGGAGTCTGAATCAAATGCCAAAGTAGCTGATGTATTGGACGTTCTAAATGAGGTAGATGAATATTCTGGTTCTTCAGAGAAAATAGACTTACTGGCCAGTGATCCTCATGAGGCTTTAATATGTAAAAGTGAAAGAGTTCACTCCAAATCAGTAGAGAGTAATATTGAAGACAAAATATTTGGGAAAACCTATCGGAAGAAGGCAAGCCTCCCCAACTTAAGCCATGTAACTGAAAATCTAATTATAGGAGCATTTGTTACTGAGCCACAGATAATACAAGAGCGTCCCCTCACAAATAAATTAAAGCGTAAAAGGAGACCTACATCAGGCCTTCATCCTGAGGATTTTATCAAGAAAGCAGATTTGGCAGTTCAAAAGACTCCTGAAATGATAAATCAGGGAACTAACCAAACGGAGCAGAATGGTCAAGTGATGAATATTACTAATAGTGGTCATGAGAATAAAACAAAAGGTGATTCTATTCAGAATGAGAAAAATCCTAACCCAATAGAATCACTCGAAAAAGAATCTGCTTTCAAAACGAAAGCTGAACCTATAAGCAGCAGTATAAGCAATATGGAACTCGAATTAAATATCCACAATTCAAAAGCACCTAAAAAGAATAGGCTGAGGAGGAAGTCTTCTACCAGGCATATTCATGCGCTTGAACTAGTAGTCAGTAGAAATCTAAGCCCACCTAATTGTACTGAATTGCAAATTGATAGTTGTTCTAGCAGTGAAGAGATAAAGAAAAAAAAGTACAACCAAATGCCAGTCAGGCACAGCAGAAACCTACAACTCATGGAAGGTAAAGAACCTGCAACTGGAGCCAAGAAGAGTAACAAGCCAAATGAACAGACAAGTAAAAGACATGACAGCGATACTTTCCCAGAGCTGAAGTTAACAAATGCACCTGGTTCTTTTACTAAGTGTTCAAATACCAGTGAACTTAAAGAATTTGTCAATCCTAGCCTTCCAAGAGAAGAAAAAGAAGAGAAACTAGAAACAGTTAAAGTGTCTAATAATGCTGAAGACCCCAAAGATCTCATGTTAAGTGGAGAAAGGGTTTTGCAAACTGAAAGATCTGTAGAGAGTAGCAGTATTTCATTGGTACCTGGTACTGATTATGGCACTCAGGAAAGTATCTCGTTACTGGAAGTTAGCACTCTAGGGAAGGCAAAAACAGAACCAAATAAATGTGTGAGTCAGTGTGCAGCATTTGAAAACCCCAAGGGACTAATTCATGGTTGTTCCAAAGATAATAGAAATGACACAGAAGGCTTTAAGTATCCATTGGGACATGAAGTTAACCACAGTCGGGAAACAAGCATAGAAATGGAAGAAAGTGAACTTGATGCTCAGTATTTGCAGAATACATTCAAGGTTTCAAAGCGCCAGTCATTTGCTCCGTTTTCAAATCCAGGAAATGCAGAAGAGGAATGTGCAACATTCTCTGCCCACTCTGGGTCCTTAAAGAAACAAAGTCCAAAAGTCACTTTTGAATGTGAACAAAAGGAAGAAAATCAAGGAAAGAATGAGTCTAATATCAAGCCTGTACAGACAGTTAATATCACTGCAGGCTTTCCTGTGGTTGGTCAGAAAGATAAGCCAGTTGATAATGCCAAATGTAGTATCAAAGGAGGCTCTAGGTTTTGTCTATCATCTCAGTTCAGAGGCAACGAAACTGGACTCATTACTCCAAATAAACATGGACTTTTACAAAACCCATATCGTATACCACCACTTTTTCCCATCAAGTCATTTGTTAAAACTAAATGTAAGAAAAATCTGCTAGAGGAAAACTTTGAGGAACATTCAATGTCACCTGAAAGAGAAATGGGAAATGAGAACATTCCAAGTACAGTGAGCACAATTAGCCGTAATAACATTAGAGAAAATGTTTTTAAAGAAGCCAGCTCAAGCAATATTAATGAAGTAGGTTCCAGTACTAATGAAGTGGGCTCCAGTATTAATGAAATAGGTTCCAGTGATGAAAACATTCAAGCAGAACTAGGTAGAAACAGAGGGCCAAAATTGAATGCTATGCTTAGATTAGGGGTTTTGCAACCTGAGGTCTATAAACAAAGTCTTCCTGGAAGTAATTGTAAGCATCCTGAAATAAAAAAGCAAGAATATGAAGAAGTAGTTCAGACTGTTAATACAGATTTCTCTCCATATCTGATTTCAGATAACTTAGAACAGCCTATGGGAAGTAGTCATGCATCTCAGGTTTGTTCTGAGACACCTGATGACCTGTTAGATGATGGTGAAATAAAGGAAGATACTAGTTTTGCTGAAAATGACATTAAGGAAAGTTCTGCTGTTTTTAGCAAAAGCGTCCAGAAAGGAGAGCTTAGCAGGAGTCCTAGCCCTTTCACCCATACACATTTGGCTCAGGGTTACCGAAGAGGGGCCAAGAAATTAGAGTCCTCAGAAGAGAACTTATCTAGTGAGGATGAAGAGCTTCCCTGCTTCCAACACTTGTTATTTGGTAAAGTAAACAATATACCTTCTCAGTCTACTAGGCATAGCACCGTTGCTACCGAGTGTCTGTCTAAGAACACAGAGGAGAATTTATTATCATTGAAGAATAGCTTAAATGACTGCAGTAACCAGGTAATATTGGCAAAGGCATCTCAGGAACATCACCTTAGTGAGGAAACAAAATGTTCTGCTAGCTTGTTTTCTTCACAGTGCAGTGAATTGGAAGACTTGACTGCAAATACAAACACCCAGGATCCTTTCTTGATTGGTTCTTCCAAACAAATGAGGCATCAGTCTGAAAGCCAGGGAGTTGGTCTGAGTGACAAGGAATTGGTTTCAGATGATGAAGAAAGAGGAACGGGCTTGGAAGAAAATAATCAAGAAGAGCAAAGCATGGATTCAAACTTAGGTGAAGCAGCATCTGGGTGTGAGAGTGAAACAAGCGTCTCTGAAGACTGCTCAGGGCTATCCTCTCAGAGTGACATTTTAACCACTCAGCAGAGGGATACCATGCAACATAACCTGATAAAGCTCCAGCAGGAAATGGCTGAACTAGAAGCTGTGTTAGAACAGCATGGGAGCCAGCCTTCTAACAGCTACCCTTCCATCATAAGTGACTCTTCTGCCCTTGAGGACCTGCGAAATCCAGAACAAAGCACATCAGAAAAAGCAGTATTAACTTCACAGAAAAGTAGTGAATACCCTATAAGCCAGAATCCAGAAGGCCTTTCTGCTGACAAGTTTGAGGTGTCTGCAGATAGTTCTACCAGTAAAAATAAAGAACCAGGAGTGGAAAGGTCATCCCCTTCTAAATGCCCATCATTAGATGATAGGTGGTACATGCACAGTTGCTCTGGGAGTCTTCAGAATAGAAACTACCCATCTCAAGAGGAGCTCATTAAGGTTGTTGATGTGGAGGAGCAACAGCTGGAAGAGTCTGGGCCACACGATTTGACGGAAACATCTTACTTGCCAAGGCAAGATCTAGAGGGAACCCCTTACCTGGAATCTGGAATCAGCCTCTTCTCTGATGACCCTGAATCTGATCCTTCTGAAGACAGAGCCCCAGAGTCAGCTCGTGTTGGCAACATACCATCTTCAACCTCTGCATTGAAAGTTCCCCAATTGAAAGTTGCAGAATCTGCCCAGAGTCCAGCTGCTGCTCATACTACTGATACTGCTGGGTATAATGCAATGGAAGAAAGTGTGAGCAGGGAGAAGCCAGAATTGACAGCTTCAACAGAAAGGGTCAACAAAAGAATGTCCATGGTGGTGTCTGGCCTGACCCCAGAAGAATTTATGCTCGTGTACAAGTTTGCCAGAAAACACCACATCACTTTAACTAATCTAATTACTGAAGAGACTACTCATGTTGTTATGAAAACAGATGCTGAGTTTGTGTGTGAACGGACACTGAAATATTTTCTAGGAATTGCGGGAGGAAAATGGGTAGTTAGCTATTTCTGGGTGACCCAGTCTATTAAAGAAAGAAAAATGCTGAATGAGCATGATTTTGAAGTCAGAGGAGATGTGGTCAATGGAAGAAACCACCAAGGTCCAAAGCGAGCAAGAGAATCCCAGGACAGAAAGATCTTCAGGGGGCTAGAAATCTGTTGCTATGGGCCCTTCACCAACATGCCCACAGATCAACTGGAATGGATGGTACAGCTGTGTGGTGCTTCTGTGGTGAAGGAGCTTTCATCATTCACCCTTGGCACAGGTGTCCACCCAATTGTGGTTGTGCAGCCAGATGCCTGGACAGAGGACAATGGCTTCCATGCAATTGGGCAGATGTGTGAGGCACCTGTGGTGACCCGAGAGTGGGTGTTGGACAGTGTAGCACTCTACCAGTGCCAGGAGCTGGACACCTACCTGATACCCCAGATCCCCCACAGCCACTACTGACTGCAGCCAGCCACAGGTACAGAGCCACAGGACCCCAAGAATGAGCTTACAAAGTGGCCTTTCCAGGCCCTGGGAGCTCCTCTCACTCTTCAGTCCTTCTACTGTCCTGGCTACTAAATATTTTATGTACATCAGCCTGAAAAGGACTTCTGGCTATGCAAGGGTCCCTTAAAGATTTTCTGCTTGAAGTCTCCCTTGGAAATCTGCCATGAGCACAAAATTATGGTAATTTTTCACCTGAGAAGATTTTAAAACCATTTAAACGCCACCAATTGAGCAAGATGCTGATTCATTATTTATCAGCCCTATTCTTTCTATTCAGGCTGTTGTTGGCTTAGGGCTGGAAGCACAGAGTGGCTTGGCCTCAAGAGAATAGCTGGTTTCCCTAAGTTTACTTCTCTAAAACCCTGTGTTCACAAAGGCAGAGAGTCAGACCCTTCAATGGAAGGAGAGTGCTTGGGATCGATTATGTGACTTAAAGTCAGAATAGTCCTTGGGCAGTTCTCAAATGTTGGAGTGGAACATTGGGGAGGAAATTCTGAGGCAGGTATTAGAAATGAAAAGGAAACTTGAAACCTGGGCATGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCAAGGTGGGCAGATCACTGGAGGTCAGGAGTTCGAAACCAGCCTGGCCAACATGGTGAAACCCCATCTCTACTAAAAATACAGAAATTAGCCGGTCATGGTGGTGGACACCTGTAATCCCAGCTACTCAGGTGGCTAAGGCAGGAGAATCACTTCAGCCCGGGAGGTGGAGGTTGCAGTGAGCCAAGATCATACCACGGCACTCCAGCCTGGGTGACAGTGAGACTGTGGCTCAAAAAAAAAAAAAAAAAAAGGAAAATGAAACTAGAAGAGATTTCTAAAAGTCTGAGATATATTTGCTAGATTTCTAAAGAATGTGTTCTAAAACAGCAGAAGATTTTCAAGAACCGGTTTCCAAAGACAGTCTTCTAATTCCTCATTAGTAATAAGTAAAATGTTTATTGTTGTAGCTCTGGTATATAATCCATTCCTCTTAAAATATAAGACCTCTGGCATGAATATTTCATATCTATAAAATGACAGATCCCACCAGGAAGGAAGCTGTTGCTTTCTTTGAGGTGATTTTTTTCCTTTGCTCCCTGTTGCTGAAACCATACAGCTTCATAAATAATTTTGCTTGCTGAAGGAAGAAAAAGTGTTTTTCATAAACCCATTATCCAGGACTGTTTATAGCTGTTGGAAGGACTAGGTCTTCCCTAGCCCCCCCAGTGTGCAAGGGCAGTGAAGACTTGATTGTACAAAATACGTTTTGTAAATGTTGTGCTGTTAACACTGCAAATAAACTTGGTAGCAAACACTTCCAAAAAAAAAAAAAAAAAA", + "expected_refget": "SQ.jj1RAXMGdOU-D39IRYnDX_fXcM7LQvb6", + "non_canonical_refgets": [ + "SQ.bh0RAohFfTjG5VAsjisxENDHZvpHIO39" + ], + "note": "cdot's genome-assembled transcript (7207 nt) instead of NCBI's record (7224 nt)." + }, + "NP_001346945.1": { + "sequence": "MAEQVALSRTQVCGILREELFQGDAFHQSDTHIFIIMGASGDLAKKKIYPTIWWLFRDGLLPENTFIVGYARSRLTVADIRKQSEPFFKATPEEKLKLEDFFARNSYVAGQYDDAASYQRLNSHMNALHLGSQANRLFYLALPPTVYEAVTKNIHESCMSQIGWNRIIVEKPFGRDLQSSDRLSNHISSLFREDQIYRIDHYLGKEMVQNLMVLRFANRIFGPIWNRDNIACVILTFKEPFGTEGRGGYFDEFGIIRDVMQNHLLQMLCLVAMEKPASTNSDDVRDEKVKVLKCISEVQANNVVLGQYVGNPDGEGEATKGYLDDPTVPRGSTTATFAAVVLYVENERWDGVPFILRCGKALNERKAEVRLQFHDVAGDIFHQQCKRNELVIRVQPNEAVYTKMMTKKPGMFFNPEESELDLTYGNRYKNVKLPDAYERLILDVFCGSQMHFVRSDELREAWRIFTPLLHQIELEKPKPIPYIYGSRGPTEADELMKRVGFQYEGTYKWVNPHKL", + "expected_refget": "SQ.j1QRpZEz0P5Ke1zpojiPxviTGGIG4dhN", + "non_canonical_refgets": [ + "SQ._KAVUSriCpoIGu4Tu7eJLZZsEPHQ9dx3" + ], + "note": "Second accession observed with two refgets on staging (2026-09-30)." + } + } +} diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index faabe6d18..f1d7b81ff 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -41,6 +41,7 @@ VALID_CHR_ACCESSION = "NC_000001.11" VALID_ACCESSION = "NM_001637.3" VALID_NT_ACCESSION = "NM_001637.3" +VALID_NT_ACCESSION_UNVERSIONED = "NM_001637" VALID_PRO_ACCESSION = "NP_001637.4" VALID_GENE = "BRCA1" VALID_UNIPROT_ACCESSION = "P05067" @@ -52,7 +53,7 @@ VALID_CLINGEN_CA_ID = "CA341478553" VALID_CLINGEN_LDH_ID = "2786738861" -VALID_MD5_DIGEST = "01234abcde%" +VALID_MD5_DIGEST = "0123456789abcdef0123456789abcdef" VALID_VMC_DIGEST = "GS_ASNKvN4=%" PRIVATE_CALIBRATION_OWNER_ID = 42 @@ -1652,6 +1653,18 @@ } +SAVED_GENERIC_DISEASE_CONCEPT = { + "type": "MappableConcept", + "conceptType": "Disease", + "name": "disease or disorder", + "primaryCoding": { + "system": "https://purl.obolibrary.org/obo/mondo.owl", + "code": "MONDO:0000001", + "iris": ["https://purl.obolibrary.org/obo/MONDO_0000001"], + }, +} + + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED = { "title": "Test BRNICH Score Calibration", "research_use_only": False, @@ -1692,6 +1705,9 @@ "primary": True, "private": False, "scoreSetId": 1, + "controls": [], + "controlsCount": 0, + "disease": SAVED_GENERIC_DISEASE_CONCEPT, "createdBy": { "recordType": "User", "firstName": TEST_USER["first_name"], @@ -1762,6 +1778,7 @@ "private": False, "urn": VALID_CALIBRATION_URN, "scoreSetId": 1, + "disease": SAVED_GENERIC_DISEASE_CONCEPT, "createdBy": { "recordType": "User", "firstName": TEST_USER["first_name"], @@ -1825,14 +1842,14 @@ TEST_SAVED_CLINVAR_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "183058", "geneSymbol": "PTEN", "clinicalSignificance": "Likely benign", "clinicalReviewStatus": "criteria provided, multiple submitters, no conflicts", "dbName": "ClinVar", "dbVersion": "11_2024", - "mappedVariants": [], + "clinvarLinks": [], } @@ -1847,14 +1864,14 @@ TEST_SAVED_GENERIC_CLINICAL_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "ABC123", "geneSymbol": "BRCA1", "clinicalSignificance": "benign", "clinicalReviewStatus": "lots of convincing evidence", "dbName": "GenDB", "dbVersion": "2024", - "mappedVariants": [], + "clinvarLinks": [], } @@ -2191,7 +2208,7 @@ "faf95MaxAncestry": TEST_GNOMAD_FAF95_MAX_ANCESTRY, "creationDate": date.today().isoformat(), "modificationDate": date.today().isoformat(), - "recordType": "GnomADVariantWithMappedVariants", + "recordType": "GnomADVariantWithVariantLinks", "id": 1, # Presuming this is the only gnomAD variant in the database } diff --git a/tests/helpers/mocks/factories.py b/tests/helpers/mocks/factories.py index 7c7f7aac7..b85f4c29f 100644 --- a/tests/helpers/mocks/factories.py +++ b/tests/helpers/mocks/factories.py @@ -8,6 +8,7 @@ from datetime import date, datetime from unittest.mock import MagicMock +from mavedb.lib.mondo import MONDO_GENERIC_CODE, MONDO_GENERIC_LABEL, MONDO_SYSTEM from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided @@ -173,11 +174,19 @@ def create_mock_score_set( ) +def create_mock_mondo_term( + code=MONDO_GENERIC_CODE, label=MONDO_GENERIC_LABEL, system=MONDO_SYSTEM, system_version=None +): + """Create a mock MondoTerm, defaulting to the generic "disease or disorder" term.""" + return create_sealed_mock(code=code, label=label, system=system, system_version=system_version) + + def create_mock_score_calibration(functional_classifications=None, primary=True, **kwargs): """Create a mock ScoreCalibration for both Pydantic validation and direct attribute access. When ``functional_classifications`` is not provided a minimal default list is - created (single not_specified classification with no ACMG). + created (single not_specified classification with no ACMG). ``disease_term`` defaults to + the generic MONDO term (calibrations always carry a non-null disease FK). """ user = kwargs.get("created_by") or create_mock_user() @@ -211,6 +220,7 @@ def create_mock_score_calibration(functional_classifications=None, primary=True, "primary": primary, "private": kwargs.get("private", not primary), "researchUseOnly": kwargs.get("research_use_only", False), + "diseaseTerm": kwargs.get("disease_term", create_mock_mondo_term()), "thresholdSources": kwargs.get("threshold_sources", []), "classificationSources": kwargs.get("classification_sources", []), "methodSources": kwargs.get("method_sources", []), diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py new file mode 100644 index 000000000..6edf490de --- /dev/null +++ b/tests/helpers/util/annotation.py @@ -0,0 +1,171 @@ +"""Seeding helpers for the new-model annotation substrate. + +The serving layer (the lean whole-set view, the ``/variants/{urn}`` detail envelope, allele +annotations, and the score-set clinical-controls routes) reads a variant's mapped alleles and +their annotations through the ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` graph and +the per-source link tables (``ClinvarAlleleLink`` / ``GnomadAlleleLink`` / ``VepAlleleConsequence``) +— not the legacy ``MappedVariant`` association tables. These helpers build that graph in tests so a +score set's variants carry alleles and annotations the way the mapping/annotation pipeline would. + +``seed_mapping_record`` is the single reusable builder; higher-level helpers (e.g. +``link_clinical_controls_to_alleles``) compose it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional, Sequence, Union + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(eq=False) +class GroupSpec: + """A ProjectionGroup row on a mapping record: one variant described at the c./g./p. levels. + + Specs that share a GroupSpec instance join that one group; its HGVS strings are written as given. + """ + + hgvs_c: Optional[str] = None + hgvs_g: Optional[str] = None + hgvs_p: Optional[str] = None + + +@dataclass +class AlleleSpec: + """One allele on a mapping record, plus the annotations that hang off it. + + Digests are the allele's identity in the serving layer: a digest already seeded reuses that allele, so give + a shared allele's annotations on one spec only. + ``group`` places the allele in a ProjectionGroup (see ``GroupSpec``). + ``clinvar_control_ids`` / ``gnomad_variant_ids`` reference rows seeded elsewhere (the router + fixtures seed ClinVar controls with ids 1 and 2); each id becomes a live link on this allele. + """ + + digest: str + level: str = "cdna" + is_authoritative: bool = False + clingen_allele_id: Optional[str] = None + hgvs_c: Optional[str] = None + hgvs_p: Optional[str] = None + hgvs_g: Optional[str] = None + post_mapped: Optional[dict] = None + vep_consequence: Optional[str] = None + group: Optional[GroupSpec] = None + clinvar_control_ids: Sequence[int] = field(default_factory=tuple) + gnomad_variant_ids: Sequence[int] = field(default_factory=tuple) + + +def seed_mapping_record( + session: Session, + variant: Union[Variant, str], + *, + alleles: Sequence[AlleleSpec], + assay_level: str = "cdna", + hgvs_assay_level: Optional[str] = None, + mapping_api_version: str = "test.0.0", + pre_mapped: Optional[dict] = None, + valid_from: Optional[datetime] = None, +) -> MappingRecord: + """Give ``variant`` a live mapping record carrying ``alleles`` and their annotations. + + ``variant`` may be a ``Variant`` instance or its URN. ``pre_mapped`` sets the record's assayed-level + VRS (the flat ``preMapped`` field on the variant detail). When ``valid_from`` is set it stamps the + record, its allele links, and every annotation link, so ``as_of`` reconstruction tests can place + the whole graph at a chosen instant (absent it, ``valid_from`` defaults to ``now()`` and the row + is live). Each distinct ``GroupSpec`` instance among the specs becomes one ProjectionGroup row with + its HGVS strings copied verbatim. Returns the created ``MappingRecord``. + """ + if isinstance(variant, str): + resolved = session.scalar(select(Variant).where(Variant.urn == variant)) + assert resolved is not None, f"variant with URN '{variant}' not found" + variant = resolved + + record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version=mapping_api_version, + pre_mapped=pre_mapped, + ) + if valid_from is not None: + record.valid_from = valid_from + session.add(record) + session.commit() + + # Keyed by instance identity so equal-looking GroupSpecs stay distinct groups. + groups: dict[int, ProjectionGroup] = {} + for spec in alleles: + if spec.group is None or id(spec.group) in groups: + continue + group = ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=spec.group.hgvs_c, + hgvs_g=spec.group.hgvs_g, + hgvs_p=spec.group.hgvs_p, + ) + if valid_from is not None: + group.valid_from = valid_from + groups[id(spec.group)] = group + + session.add_all(groups.values()) + session.flush() + + # Alleles are deduplicated by digest, so one seeded under the same digest before is reused (an allele + # several records share, like a protein consequence). + for spec in alleles: + allele = session.scalar(select(Allele).where(Allele.vrs_digest == spec.digest)) + if allele is None: + allele = Allele( + vrs_digest=spec.digest, + level=spec.level, + clingen_allele_id=spec.clingen_allele_id, + hgvs_c=spec.hgvs_c, + hgvs_p=spec.hgvs_p, + hgvs_g=spec.hgvs_g, + post_mapped=spec.post_mapped if spec.post_mapped is not None else {"type": "Allele"}, + ) + session.add(allele) + session.commit() + + links: list[Any] = [ + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=spec.is_authoritative, + projection_group=groups[id(spec.group)].id if spec.group is not None else None, + ) + ] + if spec.vep_consequence is not None: + links.append( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=spec.vep_consequence, + source_version="116", + access_date="2026-01-01", + ) + ) + links.extend(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=cid) for cid in spec.clinvar_control_ids) + links.extend(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gid) for gid in spec.gnomad_variant_ids) + + if valid_from is not None: + for link in links: + link.valid_from = valid_from + session.add_all(links) + + session.commit() + return record diff --git a/tests/helpers/util/query_plan.py b/tests/helpers/util/query_plan.py new file mode 100644 index 000000000..ea9cd9afc --- /dev/null +++ b/tests/helpers/util/query_plan.py @@ -0,0 +1,57 @@ +"""Query-plan assertions for reads that must stay O(variants) regardless of planner statistics. + +A score-set read that joins a large per-variant table can flip from per-row index lookups to a hash join +over the whole table once the planner's row estimate crosses its cost threshold. These helpers EXPLAIN a +captured statement with nested loops and sequential scans disabled, so any join the planner *could* turn +into a whole-table read does, and report scans of the given tables that are not keyed on the outer row. +""" + +from contextlib import contextmanager +from typing import Iterator, Mapping + +from sqlalchemy import event, text + + +@contextmanager +def captured_statements(session) -> Iterator[list[tuple[str, object]]]: + """Collect every ``(statement, parameters)`` the session's engine sends while the block runs.""" + captured: list[tuple[str, object]] = [] + engine = session.get_bind() + + def capture(_conn, _cursor, statement, parameters, _context, _executemany): + captured.append((statement, parameters)) + + event.listen(engine, "before_cursor_execute", capture) + try: + yield captured + finally: + event.remove(engine, "before_cursor_execute", capture) + + +def unkeyed_scans(session, statement: str, parameters, keys: Mapping[str, str]) -> list[str]: + """Scans of the tables in ``keys`` whose condition does not constrain the mapped key column. + + ``keys`` maps a table name to the column that must appear as `` =`` in the scan's index or + recheck condition, i.e. the scan is a lookup driven by the outer row rather than a read of the table. + Runs inside the caller's transaction; the ``SET LOCAL``s end with it. + """ + session.execute(text("SET LOCAL enable_nestloop = off")) + session.execute(text("SET LOCAL enable_seqscan = off")) + cursor = session.connection().connection.cursor() + cursor.execute("EXPLAIN (FORMAT JSON) " + statement, parameters) + plan = cursor.fetchone()[0][0]["Plan"] + + def walk(node): + yield node + for child in node.get("Plans", []): + yield from walk(child) + + bad = [] + for node in walk(plan): + table = node.get("Relation Name") + if table not in keys: + continue + condition = node.get("Index Cond", "") + node.get("Recheck Cond", "") + if f"{keys[table]} =" not in condition: + bad.append(f"{node['Node Type']} on {table} ({condition or 'no condition'})") + return bad diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index b6d7801ae..7a0f3df46 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -1,3 +1,4 @@ +import re from copy import deepcopy from datetime import date from typing import Any, Dict, Optional @@ -8,8 +9,6 @@ from fastapi.testclient import TestClient from sqlalchemy import select -from mavedb.models.clinical_control import ClinicalControl as ClinicalControlDbModel -from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel @@ -24,6 +23,7 @@ TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, ) +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record from tests.helpers.util.variant import mock_worker_variant_insertion @@ -204,60 +204,164 @@ def create_acc_score_set_with_variants( return score_set -def link_clinical_controls_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) +def seed_annotation_substrate(db, score_set, *, skip_first=False, pre_mapped=None): + """Give a score set's variants a live ``MappingRecord`` with one authoritative allele. + + The VA endpoints build a ``VariantAnnotationContext`` from the ``MappingRecord`` / ``Allele`` + substrate (not the legacy ``MappedVariant``), so a variant needs a live record with an + authoritative ``post_mapped`` allele before a study result / statement can be constructed. This + mirrors what the mapping pipeline writes. (Global seeding from ``mock_worker_vrs_mapping`` is + deferred to Slice 5.2/5.3, where it can be reconciled with the clinical-controls seeding.) + + ``pre_mapped`` sets the assayed-level VRS on each record (the flat ``preMapped`` on the variant + detail). With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the + "some variants were not mapped" cases). Returns the score set's variants in id order. + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() + for variant in variants[1:] if skip_first else variants: + seed_mapping_record( + db, + variant, + pre_mapped=pre_mapped, + alleles=[ + AlleleSpec( + digest=f"va-allele-{variant.id}", + is_authoritative=True, + clingen_allele_id=f"CA{variant.id}", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + return variants - # The first mapped variant gets the clinvar control, the second gets the generic control. - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) - ) - mapped_variants[1].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 2)) - ) - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() +def shift_hgvs_position(hgvs, offset): + """Move the first position in an HGVS expression by ``offset``; ``None`` and offset 0 pass through.""" + if hgvs is None or offset == 0: + return hgvs + + # DNA positions follow the type letter directly (g.123A>G); a protein position follows a three-letter + # residue (p.Gly123Arg). + return re.sub( + r"(:[gcpn]\.\(?(?:[A-Za-z]{3})?)(\d+)", lambda m: f"{m.group(1)}{int(m.group(2)) + offset}", hgvs, count=1 + ) -def link_clinvar_control_to_mapped_variant(db, score_set): - """Link the seeded ClinVar clinical control (id=1) to the first mapped variant of a score set.""" - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) +def seed_csv_substrate( + db, + score_set, + *, + assay_level="genomic", + hgvs_g=None, + hgvs_c=None, + hgvs_p=None, + vep_consequence=None, + clingen_allele_id=None, + gnomad_variant_ids=(), + clinvar_control_ids=(), + annotate="all", + valid_from=None, +): + """Seed every variant of a score set with a live mapping record on the allele substrate, shaped for the + CSV export reader (``get_score_set_variants_as_csv``). + + The authoritative allele sits at ``assay_level`` and carries the annotations the ``vep``/``clingen``/ + ``gnomad``/``clinvar`` namespaces read (VEP consequence, ClinGen id, gnomAD/ClinVar links). For a + nucleotide assay, a projection-group sibling at the other nucleotide level and a protein apex allele are + added so the ``post_mapped_hgvs_{g,c,p}`` triple reconstructs the same way the lean view does. ``hgvs_g``/ + ``hgvs_c``/``hgvs_p`` supply the canonical string at each level (the one at ``assay_level`` is also the + record's ``hgvs_assay_level``). ``annotate`` controls which variants get the authoritative-allele + annotations: ``"all"`` (every variant) or ``"first"`` (only the first, for "one linked control" cases). + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() + hgvs_g_in, hgvs_c_in, hgvs_p_in = hgvs_g, hgvs_c, hgvs_p + + for index, variant in enumerate(variants): + # Distinct variants are distinct alleles, and alleles are unique per HGVS expression, so each + # variant gets its own position. The first variant keeps the HGVS exactly as given. + hgvs_g, hgvs_c, hgvs_p = (shift_hgvs_position(h, index) for h in (hgvs_g_in, hgvs_c_in, hgvs_p_in)) + level_hgvs = {"genomic": hgvs_g, "cdna": hgvs_c, "protein": hgvs_p} + assay_hgvs = level_hgvs[assay_level] + annotated = annotate == "all" or index == 0 + # A protein assay's group carries only the protein string (no c/g fan-out). + measured = ( + GroupSpec(hgvs_g=hgvs_g, hgvs_c=hgvs_c, hgvs_p=hgvs_p) + if assay_level in ("genomic", "cdna") + else GroupSpec(hgvs_p=hgvs_p) + ) + specs = [ + AlleleSpec( + digest=f"csv-auth-{variant.id}", + level=assay_level, + is_authoritative=True, + group=measured, + hgvs_g=hgvs_g if assay_level == "genomic" else None, + hgvs_c=hgvs_c if assay_level == "cdna" else None, + hgvs_p=hgvs_p if assay_level == "protein" else None, + vep_consequence=vep_consequence if annotated else None, + clingen_allele_id=clingen_allele_id if annotated else None, + gnomad_variant_ids=tuple(gnomad_variant_ids) if annotated else (), + clinvar_control_ids=tuple(clinvar_control_ids) if annotated else (), + ) + ] + # Nucleotide assay: add the other-nucleotide projection sibling and the protein apex so the + # g/c/p triple reconstructs. A protein assay populates only the protein slot (no c/g fan-out). + if assay_level in ("genomic", "cdna"): + other_level = "cdna" if assay_level == "genomic" else "genomic" + if level_hgvs[other_level] is not None: + specs.append( + AlleleSpec( + digest=f"csv-sib-{variant.id}", + level=other_level, + group=measured, + hgvs_g=hgvs_g if other_level == "genomic" else None, + hgvs_c=hgvs_c if other_level == "cdna" else None, + ) + ) + if hgvs_p is not None: + specs.append(AlleleSpec(digest=f"csv-prot-{variant.id}", level="protein", hgvs_p=hgvs_p)) + + seed_mapping_record( + db, variant, assay_level=assay_level, hgvs_assay_level=assay_hgvs, alleles=specs, valid_from=valid_from + ) - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) - ) + return variants - db.add(mapped_variants[0]) - db.commit() +def link_clinical_controls_to_alleles(db, score_set): + """Seed the new-model annotation graph for a score set's first two variants and link the two + seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. -def link_gnomad_variants_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) + The first variant's allele gets the ClinVar control (id 1), the second gets the generic control + (id 2) — mirroring the two-control shape the clinical-controls routes are asserted against. + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() - # The first mapped variant gets the gnomAD variant. - mapped_variants[0].gnomad_variants.append( - db.scalar(select(GnomADVariantDbModel).where(GnomADVariantDbModel.id == 1)) + seed_mapping_record( + db, + variants[0], + alleles=[AlleleSpec(digest="clinical-control-allele-0", is_authoritative=True, clinvar_control_ids=[1])], + ) + seed_mapping_record( + db, + variants[1], + alleles=[AlleleSpec(digest="clinical-control-allele-1", is_authoritative=True, clinvar_control_ids=[2])], ) - - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() def mock_worker_vrs_mapping(client, db, score_set, alleles=True): diff --git a/tests/helpers/util/setup/worker.py b/tests/helpers/util/setup/worker.py index cb6949b4e..252471ca8 100644 --- a/tests/helpers/util/setup/worker.py +++ b/tests/helpers/util/setup/worker.py @@ -4,13 +4,13 @@ from sqlalchemy import select +from mavedb.models.enums.job_pipeline import JobStatus from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant from mavedb.worker.jobs import ( create_variants_for_score_set, map_variants_for_score_set, ) -from mavedb.models.enums.job_pipeline import JobStatus from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import ( TEST_CODING_LAYER, @@ -23,6 +23,16 @@ ) +# Placeholder reference accessions for qualifying target-relative HGVS in mock +# post-mapped output, keyed by the HGVS kind prefix (``c.``/``n.``/``g.``/``p.``). +_PLACEHOLDER_ACCESSIONS = { + "c.": "NM_999999.1", + "n.": "NR_999999.1", + "g.": "NC_999999.1", + "p.": "NP_999999.1", +} + + async def create_variants_in_score_set( session, mock_s3_client, score_df, count_df, mock_worker_ctx, variant_creation_run ): @@ -128,6 +138,10 @@ async def construct_mock_mapping_output( "target_gene_identifier": target.name, "alignment_level": layer, "preferred": idx == 0, + # The accession this level was aligned against — a transcript (NM_) + # only at the coding level. Reverse translation resolves its coding + # transcript hint from the cdna entry's reference_accession. + "reference_accession": _PLACEHOLDER_ACCESSIONS[f"{layer}."], "tool_name": "dcd-mapping", "tool_version": "pytest.0.0", "tool_parameters": {}, @@ -155,15 +169,26 @@ async def construct_mock_mapping_output( "alignment_level": default_alignment_level, } - # Don't alter HGVS strings in post mapped output. This makes it considerably - # easier to assert correctness in tests. + # Reuse the variant's own HGVS in post-mapped output to keep assertions simple. + # Real mapper post-mapped HGVS is always accession-qualified (it is mapped onto + # a reference sequence), so prefix a placeholder accession when the source HGVS + # is target-relative (a bare ``c.``/``n.``/``g.``/``p.`` with no accession). if with_post_mapped: - mapped_score["post_mapped"]["expressions"][0]["value"] = variant.hgvs_nt or variant.hgvs_pro + hgvs = variant.hgvs_nt or variant.hgvs_pro + if hgvs and ":" not in hgvs: + hgvs = f"{_PLACEHOLDER_ACCESSIONS.get(hgvs[:2], 'NM_999999.1')}:{hgvs}" + mapped_score["post_mapped"]["expressions"][0]["value"] = hgvs # Skip every other variant if not with_all_variants if not with_all_variants and idx % 2 == 0: mapped_score["post_mapped"] = {} + # Mirror the mapper's per-record outcome: both alleles present -> MAPPED, else + # FAILED. Tests needing benign outcomes set ``outcome`` explicitly afterward. + mapped_score["outcome"] = ( + "mapped" if mapped_score["pre_mapped"] and mapped_score["post_mapped"] else "failed" + ) + mapping_output["mapped_scores"].append(mapped_score) if not mapping_output["mapped_scores"]: diff --git a/tests/helpers/util/variant.py b/tests/helpers/util/variant.py index eede1e610..f2ef94890 100644 --- a/tests/helpers/util/variant.py +++ b/tests/helpers/util/variant.py @@ -112,7 +112,7 @@ def mock_worker_variant_insertion( ) variants = create_variants_data(scores, counts, None) num_variants = create_variants(db, item, variants) - assert num_variants == 3 + assert num_variants == len(scores) item.processing_state = ProcessingState.success item.dataset_columns = DatasetColumnsCreate( diff --git a/tests/helpers/variant_shapes.py b/tests/helpers/variant_shapes.py index 1d60c7e26..555cd974c 100644 --- a/tests/helpers/variant_shapes.py +++ b/tests/helpers/variant_shapes.py @@ -38,6 +38,8 @@ from dataclasses import dataclass, field from typing import Any, Callable, Optional +from mavedb.lib.csv.specs import CsvMappedRow +from mavedb.lib.variants import get_id_from_post_mapped from tests.helpers.constants import ( TEST_VALID_POST_MAPPED_VRS_ALLELE, TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY, @@ -188,6 +190,26 @@ def _make_non_coding(mapped_variant) -> None: ] +def to_csv_mapped_row(mapped_variant: Any) -> CsvMappedRow: + """Adapt a mock ``MappedVariant`` (as built by :meth:`VariantShape.build`) into the ``CsvMappedRow`` + the CSV composer now consumes. + + Production resolves ``hgvs_g``/``hgvs_c``/``hgvs_p``/``vrs_digest`` at the fetch layer + (``mapped_hgvs_by_level`` and the substrate query), not in the composer itself. This mirrors that + resolution over a mock, so the shape list still exercises every ``post_mapped`` payload variant for + the one CSV column that still derives from it (``post_mapped_vrs_id``). + """ + return CsvMappedRow( + hgvs_g=mapped_variant.hgvs_g, + hgvs_c=mapped_variant.hgvs_c, + hgvs_p=mapped_variant.hgvs_p, + hgvs_assay_level=mapped_variant.hgvs_assay_level, + vrs_digest=get_id_from_post_mapped(mapped_variant.post_mapped) if mapped_variant.post_mapped else None, + vep_functional_consequence=mapped_variant.vep_functional_consequence, + clingen_allele_id=mapped_variant.clingen_allele_id, + ) + + def shape_ids() -> list[str]: """Shape names, for use as pytest parametrize ids.""" return [shape.name for shape in VARIANT_SHAPES] diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index fbdbf052d..7d16ba48b 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -5,6 +5,7 @@ including mock objects with proper calibrations and configurations. """ +from types import SimpleNamespace from unittest.mock import Mock import pytest @@ -16,6 +17,19 @@ create_mock_mapped_variant_with_pathogenicity_calibration_score_set, ) +# VariantAnnotationContext (via mavedb.lib.cat_vrs) needs logging.context, which pulls in fastapi, an +# optional "server" extra. Every test module in this directory that uses the fixtures built from it +# already requires psycopg2 (also a "server" extra) for its DB session fixture, so this costs nothing +# under "core dependencies" CI beyond what those tests already needed — but without the fallback, a +# missing fastapi crashes collection for the whole directory, including files like test_flatten.py that +# need neither. +try: + from mavedb.lib.annotation.context import VariantAnnotationContext + from mavedb.lib.vrs import vrs_object_from_mapped_variant +except ImportError: + VariantAnnotationContext = None + vrs_object_from_mapped_variant = None + # Permission related helpers coupled to logging context. try: from .conftest_optional import * # noqa: F403 @@ -36,6 +50,32 @@ def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID return mapped_variant +def annotation_context_for(mapped_variant, subject_variant=None) -> VariantAnnotationContext: + """Build a DB-free ``VariantAnnotationContext`` around a mock mapped variant. + + The annotation builders read a context's ``variant`` (scores/calibrations), ``record`` (VRS-mapper + provenance), ``measured_allele`` (study-result focus + ClinGen IRI), and ``subject_variant`` (the VA + proposition subject). We synthesize the record/allele from the mock's mapping fields; the subject + defaults to the concrete measured variation, matching a non-projected variant. Mutating + ``context.variant`` mutates the underlying mock variant, since it is the same object. + """ + record = SimpleNamespace( + mapping_api_version=mapped_variant.mapping_api_version, + mapped_date=mapped_variant.mapped_date, + ) + measured_allele = SimpleNamespace( + post_mapped=mapped_variant.post_mapped, + clingen_allele_id=mapped_variant.clingen_allele_id, + ) + return VariantAnnotationContext( + variant=mapped_variant.variant, + record=record, + measured_allele=measured_allele, + subject_variant=subject_variant or vrs_object_from_mapped_variant(mapped_variant.post_mapped), + as_of=None, + ) + + @pytest.fixture def mock_mapped_variant(): """Override main fixture with properly configured mock for annotation tests.""" @@ -52,3 +92,25 @@ def mock_mapped_variant_with_functional_calibration_score_set(): def mock_mapped_variant_with_pathogenicity_calibration_score_set(): """Fixture for mock mapped variant with pathogenicity calibration score set.""" return create_mock_mapped_variant_with_pathogenicity_calibration_score_set(clingen_allele_id="CA123456") + + +@pytest.fixture +def mock_annotation_context(mock_mapped_variant) -> VariantAnnotationContext: + """A DB-free annotation context for an uncalibrated variant.""" + return annotation_context_for(mock_mapped_variant) + + +@pytest.fixture +def mock_annotation_context_with_functional_calibration_score_set( + mock_mapped_variant_with_functional_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a functional calibration.""" + return annotation_context_for(mock_mapped_variant_with_functional_calibration_score_set) + + +@pytest.fixture +def mock_annotation_context_with_pathogenicity_calibration_score_set( + mock_mapped_variant_with_pathogenicity_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a pathogenicity calibration.""" + return annotation_context_for(mock_mapped_variant_with_pathogenicity_calibration_score_set) diff --git a/tests/lib/annotation/conftest_optional.py b/tests/lib/annotation/conftest_optional.py index 143317ad4..e0cca139a 100644 --- a/tests/lib/annotation/conftest_optional.py +++ b/tests/lib/annotation/conftest_optional.py @@ -1,6 +1,6 @@ from unittest.mock import Mock -from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME +from mavedb.lib.annotation.calibration import CALIBRATION_SCOPE_EXTENSION_NAME from mavedb.lib.permissions.principal import Principal from mavedb.models.enums.user_role import UserRole from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index cbfc19f2a..c7e2215ad 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -2,12 +2,15 @@ Tests for mavedb.lib.annotation.annotate module. This module tests the main annotation functions that create statements and study results -for variants, focusing on object structure and validation. +for variants, focusing on object structure and validation. The builders take a DB-free +``VariantAnnotationContext`` (assembled by the ``mock_annotation_context*`` fixtures); the real +substrate-backed context build is covered by the integration tests in ``test_util.py``. """ # ruff: noqa: E402 from copy import deepcopy +from unittest.mock import patch import pytest @@ -20,16 +23,23 @@ variant_pathogenicity_statement, variant_study_result, ) -from tests.lib.annotation.conftest import admin_principal, make_private, owner_principal, scope_of +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from tests.lib.annotation.conftest import ( + admin_principal, + annotation_context_for, + make_private, + owner_principal, + scope_of, +) @pytest.mark.unit class TestVariantStudyResult: """Unit tests for variant study result creation.""" - def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): + def test_variant_study_result_creates_valid_result(self, mock_annotation_context): """Test that variant study result creates a valid result object.""" - result = variant_study_result(mock_mapped_variant) + result = variant_study_result(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" @@ -37,85 +47,84 @@ def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): def test_a_study_result_discloses_a_calibration_scope(self, mock_mapped_variant): # Emitted unconditionally so that a record with no scope is never ambiguous between "public" and # "produced before disclosure existed". - assert scope_of(variant_study_result(mock_mapped_variant)) == "public" + assert scope_of(variant_study_result(annotation_context_for(mock_mapped_variant))) == "public" @pytest.mark.unit class TestVariantFunctionalImpactStatement: """Unit tests for variant functional impact statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_functional_impact_statement(mock_mapped_variant) + result = variant_functional_impact_statement(mock_annotation_context) assert result is None def test_only_research_use_only_calibrations_returns_none( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_functional_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_functional_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_functional_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + context = mock_annotation_context_with_functional_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_functional_impact_statement(context) assert result is None - def test_valid_statement_creation(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_valid_statement_creation(self, mock_annotation_context_with_functional_calibration_score_set): """Test creating valid functional impact statement with proper structure.""" - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" assert all(evidence_item.type == "EvidenceLine" for evidence_item in result.hasEvidenceLines) assert all( - study_result.root.type == "ExperimentalVariantFunctionalImpactStudyResult" + study_result.type == "ExperimentalVariantFunctionalImpactStudyResult" for evidence_line in [evidence_line for evidence_line in result.hasEvidenceLines] for study_result in evidence_line.hasEvidenceItems ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that research-use-only calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_indeterminate( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that variant not in any functional range gets INDETERMINATE classification.""" from unittest.mock import patch from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set + context = mock_annotation_context_with_functional_calibration_score_set # Mock functional_classification_of_variant to return None range (variant not in any range) with patch( "mavedb.lib.annotation.annotate.functional_classification_of_variant", return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ): - result = variant_functional_impact_statement(mapped_variant) + result = variant_functional_impact_statement(context) assert result is not None assert result.type == "Statement" @@ -126,32 +135,34 @@ def test_no_statement_is_built_from_a_private_calibration( self, mock_mapped_variant_with_functional_calibration_score_set ): """A private calibration's thresholds and baseline scores must not reach an anonymous caller.""" - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_functional_calibration_score_set)) - assert variant_functional_impact_statement(mapped_variant) is None + assert variant_functional_impact_statement(context) is None def test_an_entitled_caller_receives_a_statement_from_a_private_calibration( self, mock_mapped_variant_with_functional_calibration_score_set ): """Viewer-scoped emission: an export shows each principal what that principal may see.""" - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_functional_calibration_score_set)) - assert variant_functional_impact_statement(mapped_variant, principal=admin_principal()) is not None + assert variant_functional_impact_statement(context, principal=admin_principal()) is not None def test_a_public_statement_discloses_a_public_calibration_scope( self, mock_mapped_variant_with_functional_calibration_score_set ): # VA-Spec statements carry no stable id, so a viewer-scoped statement must say that it is one. - statement = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + statement = variant_functional_impact_statement( + annotation_context_for(mock_mapped_variant_with_functional_calibration_score_set) + ) assert scope_of(statement) == "public" def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( self, mock_mapped_variant_with_functional_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_functional_calibration_score_set)) - statement = variant_functional_impact_statement(mapped_variant, principal=admin_principal()) + statement = variant_functional_impact_statement(context, principal=admin_principal()) assert scope_of(statement) == "restricted" @@ -160,49 +171,52 @@ def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( class TestVariantPathogenicityStatement: """Unit tests for variant pathogenicity statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_pathogenicity_statement(mock_mapped_variant) + result = variant_pathogenicity_statement(mock_annotation_context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_pathogenicity_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = mock_annotation_context_with_pathogenicity_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_pathogenicity_statement(context) assert result is None def test_only_research_use_only_calibration_returns_none( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_no_acmg_classifications_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_acmg_classifications_returns_none( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test that statement returns None when no ACMG classifications exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Remove ACMG classifications from all calibrations - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: acmg_removed = [deepcopy(r) for r in calibration.functional_classifications] for functional_classification in acmg_removed: functional_classification["acmgClassification"] = None calibration.functional_classifications = acmg_removed - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_valid_pathogenicity_statement_creation( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test creating valid pathogenicity statement with proper structure.""" - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.proposition.type == "VariantPathogenicityProposition" @@ -220,33 +234,33 @@ def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_p assert all(s.type == "Statement" for s in statements) assert all(ei.type == "EvidenceLine" for ei in functional_evidence_lines) assert all( - study_result.root.type == "ExperimentalVariantFunctionalImpactStudyResult" + study_result.type == "ExperimentalVariantFunctionalImpactStudyResult" for evidence_item in functional_evidence_lines for study_result in evidence_item.hasEvidenceItems ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that research-use-only pathogenicity calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_functional_annotation( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that functional annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid: no functional classifications @@ -254,20 +268,19 @@ def test_skips_invalid_calibrations_when_functional_annotation( # Valid: retain default functional classifications mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_pathogenicity_annotation( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that pathogenicity annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid calibration: no functional classifications @@ -276,17 +289,15 @@ def test_skips_invalid_calibrations_when_pathogenicity_annotation( # Valid calibration retained mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_uncertain_significance( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that variant not in any range gets UNCERTAIN_SIGNIFICANCE classification.""" from unittest.mock import patch @@ -295,7 +306,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Mock both classification functions to return None range (variant not in any range) with ( @@ -304,11 +315,11 @@ def test_variant_not_in_any_range_returns_uncertain_significance( return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ), patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), ), ): - result = variant_pathogenicity_statement(mapped_variant) + result = variant_pathogenicity_statement(context) assert result is not None assert result.type == "Statement" @@ -316,7 +327,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( assert result.classification.primaryCoding.code.root == "uncertain significance" def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Regression test: hasEvidenceItems on VariantPathogenicityEvidenceLine must be model instances. @@ -324,7 +335,7 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( VariantPathogenicityEvidenceLine validation to fail when reconstructing nested VRS objects (e.g. Allele with production genomic coordinates). Model instances must be stored directly. """ - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None for evidence_line in result.hasEvidenceLines: @@ -340,23 +351,23 @@ def test_no_statement_is_built_from_a_private_calibration( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): """A private calibration's ACMG criteria must not reach an anonymous caller.""" - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - assert variant_pathogenicity_statement(mapped_variant) is None + assert variant_pathogenicity_statement(context) is None def test_the_owner_receives_a_statement_from_their_private_calibration( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - assert variant_pathogenicity_statement(mapped_variant, principal=owner_principal()) is not None + assert variant_pathogenicity_statement(context, principal=owner_principal()) is not None def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - statement = variant_pathogenicity_statement(mapped_variant, principal=admin_principal()) + statement = variant_pathogenicity_statement(context, principal=admin_principal()) assert scope_of(statement) == "restricted" @@ -365,44 +376,49 @@ def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( class TestVariantHighestLevelAnnotation: """Unit tests for the highest-materialized-layer resolver used by the public data dump.""" - def test_study_result_when_uncalibrated(self, mock_mapped_variant): - result = variant_highest_level_annotation(mock_mapped_variant) + def test_study_result_when_uncalibrated(self, mock_annotation_context): + result = variant_highest_level_annotation(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" - def test_functional_statement_when_functional_only(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_functional_statement_when_functional_only( + self, mock_annotation_context_with_functional_calibration_score_set + ): # The functional calibration fixture has no ACMG classifications, so the variant qualifies for the # functional layer but not pathogenicity. - result = variant_highest_level_annotation(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "ExperimentalVariantFunctionalImpactProposition" def test_pathogenicity_statement_when_calibrated( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): - result = variant_highest_level_annotation(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "VariantPathogenicityProposition" - def test_none_when_unmapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - result = variant_highest_level_annotation(mock_mapped_variant) - assert result is None + def test_none_when_the_variant_has_nothing_to_annotate(self, mock_annotation_context): + """An unmapped variant yields no context at all, so it never reaches a builder. What this guards + is the other absence: one raised part-way through a build must degrade to None, not propagate.""" + with patch( + "mavedb.lib.annotation.annotate.can_annotate_variant_for_pathogenicity_evidence", + side_effect=MappingDataDoesntExistException("no mapping data"), + ): + assert variant_highest_level_annotation(mock_annotation_context) is None def test_degrades_to_a_study_result_when_the_calibration_is_private( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): # A study result reports the measured score, which publishing the score set did make public. The # variant is still described; only the calibration-derived interpretation is withheld. - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - result = variant_highest_level_annotation(mapped_variant) + result = variant_highest_level_annotation(context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" @@ -410,9 +426,9 @@ def test_degrades_to_a_study_result_when_the_calibration_is_private( def test_reaches_the_statement_layer_for_an_entitled_caller( self, mock_mapped_variant_with_pathogenicity_calibration_score_set ): - mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = annotation_context_for(make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set)) - result = variant_highest_level_annotation(mapped_variant, principal=admin_principal()) + result = variant_highest_level_annotation(context, principal=admin_principal()) assert result is not None assert result.type != "ExperimentalVariantFunctionalImpactStudyResult" diff --git a/tests/lib/annotation/test_calibration.py b/tests/lib/annotation/test_calibration.py new file mode 100644 index 000000000..576f8d39f --- /dev/null +++ b/tests/lib/annotation/test_calibration.py @@ -0,0 +1,653 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.calibration — calibration eligibility + strongest-evidence selection.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.calibration import ( + calibrations_available_for_annotation, + score_calibration_may_be_used_for_annotation, + select_strongest_functional_calibration, + select_strongest_pathogenicity_calibration, +) +from mavedb.lib.permissions.principal import Principal +from tests.lib.annotation.conftest import admin_principal, make_private + + +def _has_calibrations_for_annotation(*args, **kwargs) -> bool: + """Whether any calibration survived both the eligibility and visibility checks. + + ``calibrations_available_for_annotation`` returns the surviving calibrations; the cases below assert + only on whether the list was empty. + """ + return bool(calibrations_available_for_annotation(*args, **kwargs)) + + +@pytest.mark.unit +class TestScoreCalibrationMayBeUsedForAnnotation: + def test_returns_false_for_research_use_only_when_not_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=False, + ) + is False + ) + + def test_returns_true_for_research_use_only_when_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=True, + ) + is True + ) + + def test_returns_false_when_functional_classifications_missing( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.functional_classifications = [] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False + + def test_returns_false_for_pathogenicity_without_acmg_classifications( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for functional_classification in acmg_removed: + functional_classification["acmgClassification"] = None + calibration.functional_classifications = acmg_removed + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False + + def test_returns_true_for_pathogenicity_with_any_acmg_classification( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True + + +@pytest.mark.unit +class TestCalibrationsAvailableForAnnotation: + """ + Unit tests for calibration availability, via the _has_calibrations_for_annotation adapter below. + This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. + """ + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_annotation_context, kind): + mock_annotation_context.variant.score_set.score_calibrations = None + assert _has_calibrations_for_annotation(mock_annotation_context, kind) is False + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_annotation_context, kind): + mock_annotation_context.variant.score_set.score_calibrations = [] + assert _has_calibrations_for_annotation(mock_annotation_context, kind) is False + + @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( + self, mock_annotation_context_with_functional_calibration_score_set, annotation_type + ): + """Test that research use only calibrations are excluded by default.""" + for ( + calibration + ) in mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations: + calibration.research_use_only = True + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, annotation_type + ) + is False + ) + + @pytest.mark.parametrize( + "kind,context_fixture", + [ + ("functional", "mock_annotation_context_with_functional_calibration_score_set"), + ("pathogenicity", "mock_annotation_context_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( + self, kind, context_fixture, request + ): + """Test that research use only calibrations are included when explicitly allowed.""" + context = request.getfixturevalue(context_fixture) + for calibration in context.variant.score_set.score_calibrations: + calibration.primary = False + calibration.research_use_only = True + + assert _has_calibrations_for_annotation(context, kind, allow_research_use_only_calibrations=True) is True + + @pytest.mark.parametrize( + "kind,context_fixture", + [ + ("functional", "mock_annotation_context_with_functional_calibration_score_set"), + ("pathogenicity", "mock_annotation_context_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( + self, kind, context_fixture, request + ): + context = request.getfixturevalue(context_fixture) + + for calibration in context.variant.score_set.score_calibrations: + calibration.functional_classifications = None + + assert _has_calibrations_for_annotation(context, kind) is False + + def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( + self, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_annotation_context_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + for fr in acmg_classification_removed: + fr["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_pathogenicity_calibration_score_set, "pathogenicity" + ) + is False + ) + + def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( + self, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_annotation_context_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + acmg_classification_removed[0]["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_pathogenicity_calibration_score_set, "pathogenicity" + ) + is True + ) + + @pytest.mark.parametrize( + "kind,context_fixture", + [ + ("functional", "mock_annotation_context_with_functional_calibration_score_set"), + ("pathogenicity", "mock_annotation_context_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( + self, kind, context_fixture, request + ): + context = request.getfixturevalue(context_fixture) + + assert _has_calibrations_for_annotation(context, kind) is True + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( + self, mock_annotation_context_with_functional_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for functional annotation.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + is True + ) + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" + calibrations = ( + mock_annotation_context_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + ) + + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_pathogenicity_calibration_score_set, "pathogenicity" + ) + is True + ) + + def test_score_range_check_handles_mixed_functional_classifications( + self, + mock_annotation_context_with_functional_calibration_score_set, + ): + """Test behavior when some calibrations have functional classifications and some don't.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + calibrations[1].functional_classifications = None + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + is True + ) + + def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( + self, + mock_annotation_context_with_functional_calibration_score_set, + ): + """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + for calibration in calibrations: + if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: + acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for fc in acmg_classification_removed: + if "acmgClassification" in fc: + fc["acmgClassification"] = None + calibration.functional_classifications = acmg_classification_removed + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "pathogenicity" + ) + is False + ) + + def test_functional_annotation_with_empty_functional_classifications_list( + self, + mock_annotation_context_with_functional_calibration_score_set, + ): + """Test that functional annotation fails when functional classifications list is empty.""" + calibrations = ( + mock_annotation_context_with_functional_calibration_score_set.variant.score_set.score_calibrations + ) + + for calibration in calibrations: + calibration.functional_classifications = [] + + assert ( + _has_calibrations_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + is False + ) + + +@pytest.mark.unit +class TestCalibrationAvailabilityIsScopedToTheCaller: + """A calibration's READ rule is stricter than its score set's, so publishing a score set does not + publish its calibrations. These cases exist because this function once read + ``score_set.score_calibrations`` directly and handed private calibrations to anyone. + """ + + def test_a_private_calibration_is_not_available_for_annotation( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + calibrations_available_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + == [] + ) + + def test_omitting_the_principal_withholds_rather_than_widens( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + # The whole design rests on an omitted principal meaning "the public". Passing an explicitly + # anonymous principal and passing none at all must agree. + make_private(mock_mapped_variant_with_functional_calibration_score_set) + context = mock_annotation_context_with_functional_calibration_score_set + + assert calibrations_available_for_annotation(context, "functional") == calibrations_available_for_annotation( + context, "functional", principal=Principal() + ) + + def test_an_entitled_caller_still_receives_a_private_calibration( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + calibrations_available_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional", principal=admin_principal() + ) + != [] + ) + + def test_a_public_calibration_is_still_available_to_anyone( + self, mock_annotation_context_with_functional_calibration_score_set + ): + # The counterweight: withholding private calibrations must not withhold what publishing released. + assert ( + calibrations_available_for_annotation( + mock_annotation_context_with_functional_calibration_score_set, "functional" + ) + != [] + ) + + +@pytest.mark.unit +class TestSelectStrongestFunctionalCalibrationUnit: + """Unit tests for select_strongest_functional_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_annotation_context): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_functional_calibration(mock_annotation_context, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_annotation_context_with_functional_calibration_score_set): + """Test that single calibration is returned.""" + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_returns_first_when_all_agree(self, mock_annotation_context_with_functional_calibration_score_set): + """Test that first calibration is returned when all have same classification.""" + context = mock_annotation_context_with_functional_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] # Should return the first one + + def test_defaults_to_normal_on_conflict(self, mock_annotation_context_with_functional_calibration_score_set): + """Test that normal classification is preferred when there are conflicts.""" + from unittest.mock import MagicMock, patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + context = mock_annotation_context_with_functional_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different classifications + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + side_effect=[ + (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), + (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), + ], + ): + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + # Should return the normal classification (second one) + assert calibration == calibration2 + assert functional_range.label == "Normal" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_annotation_context_with_functional_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + # Mock to return None range but INDETERMINATE classification (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), + ): + calibration, functional_range = select_strongest_functional_calibration(context, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None + + +@pytest.mark.unit +class TestSelectStrongestPathogenicityCalibrationUnit: + """Unit tests for select_strongest_pathogenicity_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_annotation_context): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_pathogenicity_calibration(mock_annotation_context, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_annotation_context_with_pathogenicity_calibration_score_set): + """Test that single calibration is returned.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_selects_strongest_evidence_strength( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test that calibration with strongest evidence is selected.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different evidence strengths + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ( + MagicMock(label="Strong"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.VERY_STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return the one with VERY_STRONG evidence + assert calibration == calibration2 + assert functional_range.label == "Strong" + + def test_defaults_to_uncertain_on_tie(self, mock_annotation_context_with_pathogenicity_calibration_score_set): + """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength but different criteria (pathogenic vs benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Pathogenic"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return first calibration but None for range to indicate uncertain significance + assert calibration == calibration1 + assert functional_range is None + + def test_returns_classification_when_all_tied_are_same_type( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test that classification is returned normally when all tied candidates are the same type.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength and same type of criteria (both benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Benign1"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign2"), + VariantPathogenicityEvidenceLine.Criterion.BP1, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return first calibration with its range (no conflict, so normal classification) + assert calibration == calibration1 + assert functional_range.label == "Benign1" + + def test_handles_none_evidence_strength(self, mock_annotation_context_with_pathogenicity_calibration_score_set): + """Test that None evidence strength is handled correctly.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibration1 = context.variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock with None and actual strength + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return the one with actual strength (second) + assert calibration == calibration2 + assert functional_range.label == "Moderate" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations + + # Mock to return None range for all calibrations (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(context, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None diff --git a/tests/lib/annotation/test_classification.py b/tests/lib/annotation/test_classification.py index 82cd90684..fdbaf23cb 100644 --- a/tests/lib/annotation/test_classification.py +++ b/tests/lib/annotation/test_classification.py @@ -63,7 +63,7 @@ def test_normal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.normal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -76,7 +76,7 @@ def test_abnormal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.ABNORMAL @@ -89,7 +89,7 @@ def test_indeterminate_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.not_specified) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -104,7 +104,7 @@ def test_variant_not_in_any_range(self): ) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -122,7 +122,7 @@ def test_multiple_ranges_returns_first_match(self): second_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -134,7 +134,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -143,7 +143,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -152,7 +152,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) @pytest.mark.unit @@ -168,7 +168,7 @@ def test_valid_pathogenicity_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -186,7 +186,7 @@ def test_moderate_plus_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) # Verify the mapped evidence strength is MODERATE, not MODERATE_PLUS assert result[0] == pathogenicity_range @@ -201,7 +201,7 @@ def test_none_acmg_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification=None) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -216,7 +216,7 @@ def test_variant_not_in_any_range(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification, variant_in_range=False) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -229,7 +229,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -238,7 +238,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -247,7 +247,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_moderate_plus_evidence_strength_maps_to_moderate(self): """Test that MODERATE_PLUS evidence strength maps to Moderate.""" @@ -258,7 +258,7 @@ def test_moderate_plus_evidence_strength_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -282,7 +282,7 @@ def test_various_valid_acmg_combinations(self, criterion, strength): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion[criterion] @@ -301,7 +301,7 @@ def test_multiple_ranges_returns_first_match(self): score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 diff --git a/tests/lib/annotation/test_condition.py b/tests/lib/annotation/test_condition.py index 817628794..0362628ce 100644 --- a/tests/lib/annotation/test_condition.py +++ b/tests/lib/annotation/test_condition.py @@ -1,12 +1,14 @@ """ -Tests for mavedb.lib.annotation.condition module. +Tests for mavedb.lib.annotation.condition. -This module tests functions for creating GA4GH Condition objects and IRIs, -specifically for generic disease conditions used in variant annotations. +A calibration's disease condition is serialized directly from its stored MONDO term (see +mavedb.lib.mondo); an unspecified disease resolves to the generic term at write time. """ # ruff: noqa: E402 +from types import SimpleNamespace + import pytest pytest.importorskip("psycopg2") @@ -15,142 +17,43 @@ from ga4gh.core.models import iriReference as IRI from ga4gh.va_spec.base.domain_entities import Condition -from mavedb.lib.annotation.condition import generic_disease_condition, generic_disease_condition_iri -from mavedb.lib.annotation.constants import GENERIC_DISEASE_MEDGEN_CODE, MEDGEN_SYSTEM - - -@pytest.mark.unit -class TestGenericDiseaseConditionIriUnit: - """Unit tests for generic_disease_condition_iri function.""" - - def test_returns_correct_iri_structure(self): - """Test that function returns proper IRI object with expected root.""" - iri = generic_disease_condition_iri() - - assert isinstance(iri, IRI) - expected_root = f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}" - assert iri.root == expected_root - - def test_uses_correct_medgen_code(self): - """Test that function uses the correct MedGen code from constants.""" - iri = generic_disease_condition_iri() - - assert GENERIC_DISEASE_MEDGEN_CODE in iri.root - assert iri.root.endswith(GENERIC_DISEASE_MEDGEN_CODE) - - def test_uses_correct_iri_format(self): - """Test that IRI follows the expected identifiers.org format.""" - iri = generic_disease_condition_iri() +from mavedb.lib.annotation.condition import calibration_disease_condition +from mavedb.lib.mondo import MONDO_GENERIC_CODE, MONDO_GENERIC_LABEL, MONDO_SYSTEM, mondo_iri +from mavedb.models.mondo_term import MondoTerm - assert iri.root.startswith("http://identifiers.org/medgen/") - assert "identifiers.org" in iri.root - def test_iri_consistency(self): - """Test that multiple calls return consistent IRI values.""" - iri1 = generic_disease_condition_iri() - iri2 = generic_disease_condition_iri() - - assert iri1.root == iri2.root +def _calibration_with_disease(term: MondoTerm) -> SimpleNamespace: + """A stand-in calibration exposing only the disease_term the condition builder reads.""" + return SimpleNamespace(disease_term=term) @pytest.mark.unit -class TestGenericDiseaseConditionUnit: - """Unit tests for generic_disease_condition function.""" - - def test_returns_correct_condition_structure(self): - """Test that function returns proper Condition object.""" - condition = generic_disease_condition() - +class TestCalibrationDiseaseConditionUnit: + """Unit tests for the calibration disease condition.""" + + def test_builds_condition_from_the_calibration_disease_term(self): + term = MondoTerm( + code="MONDO:0015263", system=MONDO_SYSTEM, system_version="2026-01-01", label="Brugada syndrome" + ) + condition = calibration_disease_condition(_calibration_with_disease(term)) assert isinstance(condition, Condition) - assert hasattr(condition, "root") - assert isinstance(condition.root, MappableConcept) - - def test_concept_type_is_disease(self): - """Test that condition has correct conceptType.""" - condition = generic_disease_condition() - - assert condition.root.conceptType == "Disease" - - def test_primary_coding_structure(self): - """Test that primary coding has correct structure and values.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding + assert isinstance(condition, MappableConcept) + assert condition.conceptType == "Disease" + assert condition.name == "Brugada syndrome" + def test_primary_coding_reflects_the_stored_term(self): + term = MondoTerm(code="MONDO:0015263", system=MONDO_SYSTEM, label="Brugada syndrome") + coding = calibration_disease_condition(_calibration_with_disease(term)).primaryCoding assert isinstance(coding, Coding) - assert coding.code.root == GENERIC_DISEASE_MEDGEN_CODE - assert coding.system == MEDGEN_SYSTEM - - def test_coding_iris_structure(self): - """Test that coding includes correct IRI list.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - + assert coding.code.root == "MONDO:0015263" + assert coding.system == MONDO_SYSTEM assert isinstance(coding.iris, list) assert len(coding.iris) == 1 - - iri = coding.iris[0] - assert isinstance(iri, IRI) - - def test_coding_iri_matches_helper_function(self): - """Test that coding IRI matches the output of helper function.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - expected_iri = generic_disease_condition_iri() - - assert len(coding.iris) == 1 - assert coding.iris[0].root == expected_iri.root - - def test_uses_correct_constants(self): - """Test that condition uses the correct constant values.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - - assert coding.code.root == GENERIC_DISEASE_MEDGEN_CODE - assert coding.system == MEDGEN_SYSTEM - - def test_iri_contains_medgen_code(self): - """Test that IRI contains the correct MedGen code.""" - condition = generic_disease_condition() - coding = condition.root.primaryCoding - iri_root = coding.iris[0].root - - expected_iri_content = f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}" - assert iri_root == expected_iri_content - - def test_condition_consistency(self): - """Test that multiple calls return consistent condition structures.""" - condition1 = generic_disease_condition() - condition2 = generic_disease_condition() - - assert condition1.root.conceptType == condition2.root.conceptType - assert condition1.root.primaryCoding.code == condition2.root.primaryCoding.code - assert condition1.root.primaryCoding.system == condition2.root.primaryCoding.system - assert condition1.root.primaryCoding.iris[0].root == condition2.root.primaryCoding.iris[0].root - - -@pytest.mark.unit -class TestConditionConsistencyUnit: - """Unit tests for cross-function consistency in condition helpers.""" - - def test_iri_function_integration_with_condition(self): - """Test that standalone IRI function produces same result as condition's IRI.""" - standalone_iri = generic_disease_condition_iri() - condition = generic_disease_condition() - condition_iri = condition.root.primaryCoding.iris[0] - - assert standalone_iri.root == condition_iri.root - - def test_complete_condition_structure_integration(self): - """Test the complete condition structure matches expected GA4GH format.""" - condition = generic_disease_condition() - - # Verify complete structure - assert condition.root.conceptType == "Disease" - assert condition.root.primaryCoding.code.root == GENERIC_DISEASE_MEDGEN_CODE - assert condition.root.primaryCoding.system == MEDGEN_SYSTEM - assert len(condition.root.primaryCoding.iris) == 1 - - # Verify IRI structure - iri = condition.root.primaryCoding.iris[0] - expected_iri_root = f"http://identifiers.org/medgen/{GENERIC_DISEASE_MEDGEN_CODE}" - assert iri.root == expected_iri_root + assert isinstance(coding.iris[0], IRI) + assert coding.iris[0].root == mondo_iri("MONDO:0015263") + + def test_generic_disease_term_yields_the_mondo_root(self): + term = MondoTerm(code=MONDO_GENERIC_CODE, system=MONDO_SYSTEM, label=MONDO_GENERIC_LABEL) + condition = calibration_disease_condition(_calibration_with_disease(term)) + assert condition.primaryCoding.code.root == MONDO_GENERIC_CODE + assert condition.name == MONDO_GENERIC_LABEL diff --git a/tests/lib/annotation/test_conformance.py b/tests/lib/annotation/test_conformance.py index ed83878ff..965951313 100644 --- a/tests/lib/annotation/test_conformance.py +++ b/tests/lib/annotation/test_conformance.py @@ -26,7 +26,7 @@ create_mock_mapped_variant_with_pathogenicity_calibration_score_set, ) from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids -from tests.lib.annotation.conftest import scope_of +from tests.lib.annotation.conftest import annotation_context_for, scope_of # Each surface is paired with the factory that can actually exercise it: a statement built on a score # set with no calibrations returns None for every shape, which would test nothing. @@ -60,7 +60,7 @@ def test_annotation_round_trips_or_is_none(self, surface, shape): """Never raises, and anything emitted can be read back.""" _, annotate, factory = surface - annotation = annotate(shape.build(factory)) + annotation = annotate(annotation_context_for(shape.build(factory))) if annotation is None: pytest.skip(f"{shape.name} produces no annotation on this surface") @@ -71,7 +71,7 @@ def test_calibration_scope_survives_the_round_trip(self, surface, shape): between "public" and "produced before disclosure existed". It has to come back too.""" _, annotate, factory = surface - annotation = annotate(shape.build(factory)) + annotation = annotate(annotation_context_for(shape.build(factory))) if annotation is None: pytest.skip(f"{shape.name} produces no annotation on this surface") @@ -82,5 +82,7 @@ def test_calibration_scope_survives_the_round_trip(self, surface, shape): def test_every_surface_emits_something_for_at_least_one_shape(): """Guards the suite above: a surface that returned None everywhere would silently skip every case.""" for name, annotate, factory in ANNOTATION_SURFACES: - emitted = [shape.name for shape in VARIANT_SHAPES if annotate(shape.build(factory)) is not None] + emitted = [ + shape.name for shape in VARIANT_SHAPES if annotate(annotation_context_for(shape.build(factory))) is not None + ] assert emitted, f"{name} emitted nothing for any shape; its conformance cases are all skips" diff --git a/tests/lib/annotation/test_constants.py b/tests/lib/annotation/test_constants.py deleted file mode 100644 index 558cc42e2..000000000 --- a/tests/lib/annotation/test_constants.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Tests for mavedb.lib.annotation.constants module. - -This module tests constant values used throughout the annotation system. -""" - -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("psycopg2") - -from mavedb.lib.annotation.constants import ( - GENERIC_DISEASE_MEDGEN_CODE, - MEDGEN_SYSTEM, -) - - -@pytest.mark.unit -class TestAnnotationConstants: - """Unit tests for annotation constants.""" - - def test_generic_disease_medgen_code(self): - """Test generic disease MedGen code constant.""" - assert GENERIC_DISEASE_MEDGEN_CODE == "C0012634" - assert isinstance(GENERIC_DISEASE_MEDGEN_CODE, str) - - def test_medgen_system(self): - """Test MedGen system URL constant.""" - expected_url = "https://www.ncbi.nlm.nih.gov/medgen/" - assert MEDGEN_SYSTEM == expected_url - assert isinstance(MEDGEN_SYSTEM, str) - assert MEDGEN_SYSTEM.startswith("https://") - assert MEDGEN_SYSTEM.endswith("/") - - def test_constants_immutability(self): - """Test that constants are properly defined as strings.""" - # These should be string constants, not mutable objects - assert isinstance(GENERIC_DISEASE_MEDGEN_CODE, str) - assert isinstance(MEDGEN_SYSTEM, str) diff --git a/tests/lib/annotation/test_context.py b/tests/lib/annotation/test_context.py new file mode 100644 index 000000000..479e8f694 --- /dev/null +++ b/tests/lib/annotation/test_context.py @@ -0,0 +1,215 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.context — the VA proposition-subject grain (Slice 5.1).""" + +from datetime import datetime, timezone +from unittest import mock + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation + +from sqlalchemy import select + +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record + + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _gnomad_variant(session, db_identifier: str) -> int: + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier=db_identifier, + db_version="v4.1", + allele_count=1, + allele_number=100, + allele_frequency=0.01, + ) + session.add(gnomad_variant) + session.commit() + return gnomad_variant.id + + +def _gnomad_codes(subject) -> list[str]: + return [ + m.coding.code.root for m in subject.mappings or [] if m.coding.system == "https://gnomad.broadinstitute.org" + ] + + +@pytest.mark.integration +class TestVariantAnnotationContextSubject: + """variant_annotation_context — the VA *proposition* subject grain (Slice 5.1). + + The subject always anchors on the **measured** allele. A protein assay unfurls the full equivalence + class of encoders; a nucleotide assay keeps its precise coordinate partner + protein consequence and + excludes the sibling encoders (distinct variants) that the reverse-translation fan left on the record. + Either way the subject is a Cat-VRS ``CategoricalVariant`` when a projection member exists; it falls + back to the concrete measured ``MolecularVariation`` when the categorical can't be assembled. + """ + + def test_protein_assay_subject_is_a_categorical_variant(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="protein", + alleles=[ + AlleleSpec( + digest="prot", level="protein", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="cdna", level="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + + def test_nucleotide_assay_subject_is_a_categorical_variant_over_the_measured_change( + self, session, setup_lib_db_with_mapped_variant + ): + """Anchored on the measured nt change: keep its projection_group coordinate partner + the protein + consequence, exclude the sibling encoder (a distinct variant, different projection group).""" + mapped_variant = setup_lib_db_with_mapped_variant + measured, sibling = GroupSpec(), GroupSpec() + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", + level="cdna", + is_authoritative=True, + group=measured, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + ), + # The measured change's precise coordinate partner (same projection group). + AlleleSpec( + digest="gen", level="genomic", group=measured, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + # A sibling encoder of the same protein consequence — distinct variant, other group: excluded. + AlleleSpec( + digest="sibling", level="cdna", group=sibling, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="prot", level="protein", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + # defining cdna + gen partner + protein apex; the sibling encoder is dropped. + assert len(context.subject_variant.members) == 3 + + def test_single_allele_subject_falls_back_to_molecular_variation(self, session, setup_lib_db_with_mapped_variant): + """A lone measured allele (no projection member) → the categorical build yields the concrete allele.""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", level="cdna", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, MolecularVariation) + assert not isinstance(context.subject_variant, CategoricalVariant) + + def test_unmapped_variant_has_no_context(self, session, setup_lib_db_with_mapped_variant): + """No live mapping record on the new substrate → no annotation context.""" + context = variant_annotation_context(session, setup_lib_db_with_mapped_variant.variant) + + assert context is None + + def test_subject_mappings_cover_only_the_narrow_members_at_as_of(self, session, setup_lib_db_with_mapped_variant): + """The subject cross-references its own members only (not the dropped sibling encoder), as of the + requested instant: a gnomAD match replaced at T1 still maps to the old ID before T1.""" + mapped_variant = setup_lib_db_with_mapped_variant + old_id = _gnomad_variant(session, "1-100-A-G") + new_id = _gnomad_variant(session, "1-100-A-T") + sibling_id = _gnomad_variant(session, "1-101-C-T") + measured, sibling = GroupSpec(), GroupSpec() + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + valid_from=T0, + alleles=[ + AlleleSpec( + digest="cdna", + level="cdna", + is_authoritative=True, + group=measured, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + ), + AlleleSpec( + digest="gen", + level="genomic", + group=measured, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + gnomad_variant_ids=[old_id], + ), + AlleleSpec( + digest="sibling", + level="cdna", + group=sibling, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + gnomad_variant_ids=[sibling_id], + ), + ], + ) + old_link = session.scalar(select(GnomadAlleleLink).where(GnomadAlleleLink.gnomad_variant_id == old_id)) + assert old_link is not None + old_link.retire(at=T1) + new_link = GnomadAlleleLink(allele_id=old_link.allele_id, gnomad_variant_id=new_id) + new_link.valid_from = T1 + session.add(new_link) + session.commit() + + past = variant_annotation_context( + session, mapped_variant.variant, as_of=datetime(2020, 6, 1, tzinfo=timezone.utc) + ) + current = variant_annotation_context(session, mapped_variant.variant) + + assert past is not None and isinstance(past.subject_variant, CategoricalVariant) + assert current is not None and isinstance(current.subject_variant, CategoricalVariant) + assert _gnomad_codes(past.subject_variant) == ["1-100-A-G"] + assert _gnomad_codes(current.subject_variant) == ["1-100-A-T"] + + def test_single_allele_subject_skips_the_cross_reference_fetch(self, session, setup_lib_db_with_mapped_variant): + """A lone measured allele is served bare, so nothing would carry the cross-references.""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", level="cdna", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + ], + ) + + with mock.patch("mavedb.lib.annotation.context.get_allele_cross_references") as fetch: + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + fetch.assert_not_called() diff --git a/tests/lib/annotation/test_contribution.py b/tests/lib/annotation/test_contribution.py index 51597db37..fbc0022b2 100644 --- a/tests/lib/annotation/test_contribution.py +++ b/tests/lib/annotation/test_contribution.py @@ -16,6 +16,7 @@ from ga4gh.core.models import Extension from ga4gh.va_spec.base import Contribution +from mavedb import __version__ from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_creator_contribution, @@ -23,14 +24,18 @@ mavedb_score_calibration_contribution, mavedb_vrs_contribution, ) +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_resource_with_dates, create_mock_score_calibration, create_mock_user, ) +from tests.lib.annotation.conftest import annotation_context_for @pytest.mark.unit @@ -56,11 +61,11 @@ def test_has_correct_activity_type(self): assert contribution.activityType == "software application programming interface" - def test_has_current_date(self): - """Test that contribution uses current date.""" + def test_carries_no_date(self): + """The software's provenance is its version, carried by the agent; serialization time is not provenance.""" contribution = mavedb_api_contribution() - assert contribution.date.date() == datetime.today().date() + assert contribution.date is None def test_has_contributor(self): """Test that contribution has a contributor.""" @@ -68,14 +73,20 @@ def test_has_contributor(self): assert contribution.contributor is not None - def test_consistency(self): - """Test that multiple calls produce consistent results.""" - contribution1 = mavedb_api_contribution() - contribution2 = mavedb_api_contribution() + def test_contributor_carries_the_api_version(self): + """The API version is what this contribution actually asserts.""" + contribution = mavedb_api_contribution() + + versions = [e.value for e in contribution.contributor.extensions if e.name == "mavedbApiVersion"] + assert versions == [__version__] + + def test_repeated_calls_are_identical(self): + """Two calls must serialize to the same bytes. - assert contribution1.name == contribution2.name - assert contribution1.description == contribution2.description - assert contribution1.activityType == contribution2.activityType + This is what makes identical provenance deduplicate in a normalized export, and what makes the + public `va.ndjson` reproducible across runs of unchanged data. + """ + assert mavedb_api_contribution() == mavedb_api_contribution() @pytest.mark.unit @@ -85,14 +96,14 @@ class TestMavedbVrsContributionUnit: def test_returns_contribution_object(self): """Test that function returns proper Contribution object.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) def test_has_correct_name_and_description(self): """Test that contribution has correct name and description.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.name == "MaveDB VRS Mapper" assert contribution.description == "Contribution from the MaveDB VRS mapping software" @@ -100,7 +111,7 @@ def test_has_correct_name_and_description(self): def test_has_correct_activity_type(self): """Test that contribution has correct activity type.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.activityType == "human genome sequence mapping process" @@ -108,14 +119,14 @@ def test_uses_mapped_variant_date(self): """Test that contribution uses mapped variant date.""" test_date = datetime(2024, 3, 15, 12, 0, 0) mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.date == test_date def test_has_contributor(self): """Test that contribution has a contributor.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.contributor is not None @@ -123,7 +134,7 @@ def test_has_contributor(self): def test_various_api_versions(self, api_version): """Test function works with various API versions.""" mapped_variant = create_mock_mapped_variant(mapping_api_version=api_version) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) assert contribution.name == "MaveDB VRS Mapper" @@ -316,7 +327,12 @@ class TestContributionIntegration: def test_contributions_with_real_db_objects(self, session, setup_lib_db_with_mapped_variant): """Test contribution creation from persisted SQLAlchemy objects.""" mapped_variant = setup_lib_db_with_mapped_variant - vrs_contribution = mavedb_vrs_contribution(mapped_variant) + # The VRS contribution derives from the record's mapper provenance, not the subject; hand the + # context an explicit subject so it doesn't depend on this fixture's minimal post_mapped. + context = annotation_context_for( + mapped_variant, subject_variant=vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + ) + vrs_contribution = mavedb_vrs_contribution(context) creator = session.query(User).first() score_set = mapped_variant.variant.score_set @@ -326,6 +342,7 @@ def test_contributions_with_real_db_objects(self, session, setup_lib_db_with_map primary=True, investigator_provided=True, private=False, + disease_term=get_generic_disease_term(session), created_by_id=creator.id, modified_by_id=creator.id, ) @@ -350,7 +367,7 @@ def test_all_contributions_return_consistent_structure(self): api_contrib = mavedb_api_contribution() mapped_variant = create_mock_mapped_variant() - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) calibration = create_mock_score_calibration() cal_contrib = mavedb_score_calibration_contribution(calibration) @@ -368,7 +385,12 @@ def test_all_contributions_return_consistent_structure(self): assert contrib.description is not None assert contrib.activityType is not None assert contrib.contributor is not None + + # A date is carried only where a stored one exists to carry. The API contribution has none: + # its provenance is the agent's version, not the moment of serialization. + for contrib in [vrs_contrib, cal_contrib, creator_contrib, modifier_contrib]: assert contrib.date is not None + assert api_contrib.date is None def test_date_formatting_consistency(self): """Test that all functions format dates consistently.""" @@ -376,7 +398,7 @@ def test_date_formatting_consistency(self): # VRS contribution mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert vrs_contrib.date == test_date # Score calibration contribution diff --git a/tests/lib/annotation/test_document.py b/tests/lib/annotation/test_document.py index 0a9b0ac86..d87e9b227 100644 --- a/tests/lib/annotation/test_document.py +++ b/tests/lib/annotation/test_document.py @@ -8,6 +8,7 @@ """ import urllib.parse +from types import SimpleNamespace import pytest @@ -19,8 +20,7 @@ from mavedb.lib.annotation.document import ( experiment_as_iri, experiment_to_document, - mapped_variant_as_iri, - mapped_variant_to_document, + measured_allele_as_iri, score_calibration_as_document, score_set_as_iri, score_set_to_document, @@ -81,42 +81,24 @@ def test_score_set_to_document(self, mock_score_set): @pytest.mark.unit -class TestMappedVariantDocumentFunctions: - """Unit tests for mapped variant document creation functions.""" +class TestMeasuredAlleleDocumentFunctions: + """Unit tests for measured-allele IRI generation.""" - def test_mapped_variant_as_iri(self, mock_mapped_variant): - """Test IRI generation for mapped variants with ClinGen allele ID.""" - expected_iri_root = ( - f"https://mavedb.org/variant/{urllib.parse.quote_plus(mock_mapped_variant.clingen_allele_id)}" - ) - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri(self): + """Test IRI generation for a measured allele with a ClinGen allele ID.""" + allele = SimpleNamespace(clingen_allele_id="CA123456") + expected_iri_root = f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}" + result = measured_allele_as_iri(allele) assert result.root == expected_iri_root - def test_mapped_variant_as_iri_no_caid(self, mock_mapped_variant): - """Test IRI generation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri_no_caid(self): + """Test IRI generation for a measured allele without a ClinGen allele ID returns None.""" + allele = SimpleNamespace(clingen_allele_id=None) + result = measured_allele_as_iri(allele) assert result is None - def test_mapped_variant_to_document(self, mock_mapped_variant): - """Test document creation for mapped variants with ClinGen allele ID.""" - document = mapped_variant_to_document(mock_mapped_variant) - - assert document.id == mock_mapped_variant.variant.urn - assert document.name == "MaveDB Mapped Variant" - assert document.documentType == "mapped genomic variant description" - assert len(document.urls) > 0 - assert mapped_variant_as_iri(mock_mapped_variant).root in document.urls - - def test_mapped_variant_to_document_no_caid(self, mock_mapped_variant): - """Test document creation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - document = mapped_variant_to_document(mock_mapped_variant) - - assert document is None - @pytest.mark.unit class TestVariantDocumentFunctions: diff --git a/tests/lib/annotation/test_eligibility.py b/tests/lib/annotation/test_eligibility.py new file mode 100644 index 000000000..9c8f2ff47 --- /dev/null +++ b/tests/lib/annotation/test_eligibility.py @@ -0,0 +1,157 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.eligibility — variant annotatability predicates.""" + +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.eligibility import ( + _can_annotate_variant_base_assumptions, + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.lib.annotation.conftest import admin_principal, annotation_context_for, make_private + + +@pytest.mark.unit +class TestBaseAnnotationAssumptionsUnit: + def test_base_assumption_check_returns_false_when_score_is_none(self, mock_annotation_context): + mock_annotation_context.variant.data = {"score_data": {"score": None}} + + assert _can_annotate_variant_base_assumptions(mock_annotation_context) is False + + def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_annotation_context): + assert _can_annotate_variant_base_assumptions(mock_annotation_context) is True + + +@pytest.mark.unit +class TestPathogenicityAnnotationEligibilityUnit: + def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_annotation_context): + with patch("mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", return_value=False): + result = can_annotate_variant_for_pathogenicity_evidence(mock_annotation_context) + + assert result is False + + def test_pathogenicity_range_check_returns_false_when_no_calibrations_available(self, mock_annotation_context): + with patch( + "mavedb.lib.annotation.eligibility.calibrations_available_for_annotation", + return_value=[], + ): + result = can_annotate_variant_for_pathogenicity_evidence(mock_annotation_context) + + assert result is False + + def test_pathogenicity_range_check_returns_true_when_all_conditions_met( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_annotation_context_with_pathogenicity_calibration_score_set + ) + is True + ) + + def test_a_private_only_calibration_is_ineligible_for_an_anonymous_caller( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_annotation_context_with_pathogenicity_calibration_score_set + ) + is False + ) + + def test_a_private_only_calibration_is_eligible_for_an_entitled_caller( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, + ): + make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_annotation_context_with_pathogenicity_calibration_score_set, principal=admin_principal() + ) + is True + ) + + +@pytest.mark.unit +class TestFunctionalAnnotationEligibilityUnit: + def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_annotation_context): + with patch( + "mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", + return_value=False, + ): + result = can_annotate_variant_for_functional_statement(mock_annotation_context) + + assert result is False + + def test_functional_range_check_returns_false_when_no_calibrations_available(self, mock_annotation_context): + with patch( + "mavedb.lib.annotation.eligibility.calibrations_available_for_annotation", + return_value=[], + ): + result = can_annotate_variant_for_functional_statement(mock_annotation_context) + + assert result is False + + def test_functional_range_check_returns_true_when_all_conditions_met( + self, mock_annotation_context_with_functional_calibration_score_set + ): + assert ( + can_annotate_variant_for_functional_statement(mock_annotation_context_with_functional_calibration_score_set) + is True + ) + + def test_a_private_only_calibration_is_ineligible_for_an_anonymous_caller( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + can_annotate_variant_for_functional_statement(mock_annotation_context_with_functional_calibration_score_set) + is False + ) + + def test_a_private_only_calibration_is_eligible_for_an_entitled_caller( + self, + mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, + ): + make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert ( + can_annotate_variant_for_functional_statement( + mock_annotation_context_with_functional_calibration_score_set, principal=admin_principal() + ) + is True + ) + + +@pytest.mark.integration +class TestAnnotationEligibilityIntegration: + def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): + # Make score presence explicit so a negative result is due to missing calibrations, and give the + # context a real VRS allele to build its subject variant from. + setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} + setup_lib_db_with_mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE + context = annotation_context_for(setup_lib_db_with_mapped_variant) + + pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence(context) + functional_allowed = can_annotate_variant_for_functional_statement(context) + + # DB fixture score sets do not include calibrations by default, so both should be False. + assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] + assert pathogenicity_allowed is False + assert functional_allowed is False diff --git a/tests/lib/annotation/test_evidence_line.py b/tests/lib/annotation/test_evidence_line.py index fb6f33b74..36144c9b6 100644 --- a/tests/lib/annotation/test_evidence_line.py +++ b/tests/lib/annotation/test_evidence_line.py @@ -21,10 +21,11 @@ from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) -from mavedb.lib.annotation.statement import mapped_variant_to_functional_statement +from mavedb.lib.annotation.statement import functional_statement +from tests.helpers.mocks.factories import create_mock_score_calibration @pytest.mark.unit @@ -49,23 +50,23 @@ class TestAcmgEvidenceLine: ) def test_acmg_evidence_line_with_met_valid_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, expected_outcome, expected_strength, expected_direction, ): """Test ACMG evidence line creation with met valid clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context, score_calibration) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) if expected_strength == StrengthOfEvidenceProvided.STRONG: expected_evidence_outcome = expected_outcome.value @@ -73,7 +74,7 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( expected_evidence_outcome = f"{expected_outcome.value}_{expected_strength.name.lower()}" assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Met" @@ -87,11 +88,11 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( def test_acmg_evidence_line_with_not_met_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Test ACMG evidence line creation with not met clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] expected_outcome = VariantPathogenicityEvidenceLine.Criterion.PS3 expected_strength = None expected_evidence_outcome = f"{expected_outcome.value}_not_met" @@ -100,13 +101,13 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context, score_calibration) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Not Met" @@ -117,19 +118,20 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( assert result.targetProposition == proposition assert len(result.hasEvidenceItems) == 1 - def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_mapped_variant): + def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_annotation_context): """Test that ACMG evidence line creation raises error when no calibrations exist.""" - mock_mapped_variant.variant.score_set.score_calibrations = None + context = mock_annotation_context + context.variant.score_set.score_calibrations = None score_calibration = MagicMock() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) - study_result = variant_study_result(mock_mapped_variant) - acmg_evidence_line(mock_mapped_variant, score_calibration, proposition, [study_result]) + proposition = variant_pathogenicity_proposition(context, create_mock_score_calibration()) + study_result = variant_study_result(context) + acmg_evidence_line(context, score_calibration, proposition, [study_result]) def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_error( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Regression test: VariantPathogenicityEvidenceLine must accept Statement instances directly. @@ -138,22 +140,22 @@ def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_err nested VRS objects from those dicts (e.g. Allele with real genomic coordinates). Evidence model instances must be passed directly, not serialized. """ - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - - study_result = variant_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + + study_result = variant_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, ExperimentalVariantFunctionalImpactClassification.NORMAL, ) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + clinical_proposition = variant_pathogenicity_proposition(context, score_calibration) - result = acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + result = acmg_evidence_line(context, score_calibration, clinical_proposition, [statement]) assert isinstance(result, VariantPathogenicityEvidenceLine) assert len(result.hasEvidenceItems) == 1 @@ -166,16 +168,16 @@ class TestFunctionalEvidenceLine: """Unit tests for functional evidence line creation.""" def test_functional_evidence_line_with_valid_functional_evidence( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test functional evidence line creation with valid evidence.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - study_result = variant_study_result(mapped_variant) - result = functional_evidence_line(mapped_variant, score_calibration, [study_result]) + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + study_result = variant_study_result(context) + result = functional_evidence_line(context, score_calibration, [study_result]) assert isinstance(result, EvidenceLine) - assert result.description == f"Functional evidence line for {mapped_variant.variant.urn}" + assert result.description == f"Functional evidence line for {context.variant.urn}" assert result.directionOfEvidenceProvided is not None assert result.specifiedBy assert result.contributions diff --git a/tests/lib/annotation/test_flatten.py b/tests/lib/annotation/test_flatten.py index 21ecfa173..7480d7e70 100644 --- a/tests/lib/annotation/test_flatten.py +++ b/tests/lib/annotation/test_flatten.py @@ -52,7 +52,7 @@ def test_pathogenicity_calibration_populates_all_fields(self): urn="urn:mavedb:calibration:1", title="Clinical Calibration" ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.functional_classification == "abnormal" assert annotation.acmg_criterion == "PS3" @@ -65,7 +65,7 @@ def test_pathogenicity_calibration_populates_all_fields(self): def test_benign_criterion_yields_benign_classification(self): calibration, mapped_variant = _pathogenicity_calibration_and_variant(criterion=ACMGCriterion.BS3) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.acmg_criterion == "BS3" assert annotation.pathogenicity_classification == "BENIGN" @@ -80,7 +80,7 @@ def test_functional_only_calibration_omits_acmg_fields(self): [classification], urn="urn:mavedb:calibration:2", title="Functional Calibration" ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.functional_classification == "normal" assert annotation.acmg_criterion is None @@ -93,7 +93,7 @@ def test_functional_only_calibration_omits_acmg_fields(self): def test_no_calibration_yields_empty_annotation(self): mapped_variant = create_mock_mapped_variant() - assert flatten_annotation(mapped_variant, None) == FlatAnnotation() + assert flatten_annotation(mapped_variant.variant, None) == FlatAnnotation() def test_calibration_without_ranges_keeps_its_identity(self): """Which calibration was consulted is known even when it can classify nothing. @@ -105,7 +105,7 @@ def test_calibration_without_ranges_keeps_its_identity(self): [], urn="urn:mavedb:calibration:3", title="Baseline Only" ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.calibration_urn == "urn:mavedb:calibration:3" assert annotation.calibration_title == "Baseline Only" @@ -119,7 +119,7 @@ def test_calibration_without_ranges_keeps_its_identity(self): def test_variant_outside_all_ranges_is_uncertain(self): calibration, mapped_variant = _pathogenicity_calibration_and_variant(variant_in_range=False) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.functional_classification == "indeterminate" assert annotation.acmg_criterion == "PS3" @@ -143,7 +143,7 @@ def test_evidence_outcome_code(self, criterion, evidence_strength, expected_code criterion=criterion, evidence_strength=evidence_strength ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.acmg_evidence_outcome_code == expected_code @@ -153,7 +153,7 @@ def test_moderate_plus_is_preserved(self): evidence_strength=StrengthOfEvidenceProvided.MODERATE_PLUS ) - annotation = flatten_annotation(mapped_variant, calibration) + annotation = flatten_annotation(mapped_variant.variant, calibration) assert annotation.acmg_evidence_strength == "MODERATE_PLUS" assert annotation.acmg_evidence_outcome_code == "PS3_moderate_plus" diff --git a/tests/lib/annotation/test_method.py b/tests/lib/annotation/test_method.py index 2d3954d29..cf3bfb67e 100644 --- a/tests/lib/annotation/test_method.py +++ b/tests/lib/annotation/test_method.py @@ -230,15 +230,12 @@ def test_functional_score_calibration_as_method_falls_back_to_not_provided(self) assert method.reportedIn.root == "Not Provided" def test_pathogenicity_score_calibration_as_method_sets_method_type(self): - """Test pathogenicity calibration method includes criterion-based methodType.""" + """A calibration's method is a functional data assessment, the type that permits PS3 and BS3.""" score_calibration = Mock() score_calibration.publication_identifier_associations = [] - method = pathogenicity_score_calibration_as_method( - score_calibration, - VariantPathogenicityEvidenceLine.Criterion.PS3, - ) + method = pathogenicity_score_calibration_as_method(score_calibration) assert method.name == "Calibration method" assert method.reportedIn.root == "Not Provided" - assert method.methodType == VariantPathogenicityEvidenceLine.Criterion.PS3.value + assert method.methodType == VariantPathogenicityEvidenceLine.MethodType.FUNCTIONAL_DATA_ASSESSMENT.value diff --git a/tests/lib/annotation/test_proposition.py b/tests/lib/annotation/test_proposition.py index cb883e0bc..9fa200f93 100644 --- a/tests/lib/annotation/test_proposition.py +++ b/tests/lib/annotation/test_proposition.py @@ -7,51 +7,175 @@ clinical and functional impact propositions. """ +from types import SimpleNamespace +from unittest.mock import patch + import pytest pytest.importorskip("psycopg2") +from ga4gh.cat_vrs.models import CategoricalVariant from ga4gh.va_spec.base.core import ( ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition, ) from ga4gh.vrs.models import MolecularVariation +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + sequence_feature_for_variant, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record_allele import MappingRecordAllele +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.helpers.mocks.factories import create_mock_mondo_term, create_mock_score_calibration +from tests.lib.annotation.conftest import annotation_context_for + + +def _protein_categorical_variant() -> CategoricalVariant: + """A Cat-VRS ``CategoricalVariant`` for a protein-measured variant (defining protein + `encodes` nt). + + Built over transient links (no DB) so the proposition tests can assert the builders accept a + categorical subject — the shape a protein assay produces under Slice 5.1. + """ + links = [ + MappingRecordAllele( + is_authoritative=True, + allele=Allele(level="protein", vrs_digest="prot", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + MappingRecordAllele( + is_authoritative=False, + allele=Allele(level="cdna", vrs_digest="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + return transit.categorical_variant @pytest.mark.unit class TestExperimentalVariantClinicalImpactProposition: """Unit tests for experimental variant clinical impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_clinical_impact_proposition(self, mock_mapped_variant): - """Test creation of clinical impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) + def test_variant_pathogenicity_proposition(self, mock_annotation_context): + """The proposition's condition defaults to the calibration's generic disease term.""" + result = variant_pathogenicity_proposition(mock_annotation_context, create_mock_score_calibration()) assert isinstance(result, VariantPathogenicityProposition) - assert result.description == f"Variant pathogenicity proposition for {mock_mapped_variant.variant.urn}." - assert isinstance(result.subjectVariant, MolecularVariation) + assert result.description == f"Variant pathogenicity proposition for {mock_annotation_context.variant.urn}." + assert isinstance(result.subject, MolecularVariation) assert result.predicate == "isCausalFor" - assert result.objectCondition.root.conceptType == "Disease" - assert result.objectCondition.root.primaryCoding.code.root == "C0012634" - assert result.objectCondition.root.primaryCoding.system == "https://www.ncbi.nlm.nih.gov/medgen/" + assert result.object.conceptType == "Disease" + assert result.object.primaryCoding.code.root == "MONDO:0000001" + assert result.object.primaryCoding.system == "https://purl.obolibrary.org/obo/mondo.owl" + + def test_clinical_impact_proposition_reflects_the_calibration_disease(self, mock_mapped_variant): + """A calibration with a specific disease term drives the proposition's condition.""" + calibration = create_mock_score_calibration( + disease_term=create_mock_mondo_term(code="MONDO:0015263", label="Brugada syndrome") + ) + result = variant_pathogenicity_proposition(annotation_context_for(mock_mapped_variant), calibration) + + assert result.object.primaryCoding.code.root == "MONDO:0015263" + assert result.object.name == "Brugada syndrome" + + def test_clinical_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_pathogenicity_proposition(context, create_mock_score_calibration()) + + assert isinstance(result.subject, CategoricalVariant) @pytest.mark.unit class TestExperimentalVariantFunctionalImpactProposition: """Unit tests for experimental variant functional impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_functional_impact_proposition(self, mock_mapped_variant): - """Test creation of functional impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_functional_impact_proposition(mock_mapped_variant) + def test_variant_functional_impact_proposition(self, mock_annotation_context): + """Test creation of functional impact proposition from a variant annotation context.""" + result = variant_functional_impact_proposition(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactProposition) - assert result.description == f"Variant functional impact proposition for {mock_mapped_variant.variant.urn}." - assert isinstance(result.subjectVariant, MolecularVariation) + assert result.description == f"Variant functional impact proposition for {mock_annotation_context.variant.urn}." + assert isinstance(result.subject, MolecularVariation) assert result.predicate == "impactsFunctionOf" - assert result.objectSequenceFeature.primaryCoding.code.root == "BRCA1" - assert result.objectSequenceFeature.primaryCoding.system == "https://www.genenames.org/" + assert result.object.primaryCoding.code.root == "BRCA1" + assert result.object.primaryCoding.system == "https://www.genenames.org/" assert result.experimentalContextQualifier is not None + + def test_functional_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_functional_impact_proposition(context) + + assert isinstance(result.subject, CategoricalVariant) + + +@pytest.mark.unit +class TestSequenceFeatureForVariantUnit: + """sequence_feature_for_variant — co-located with the propositions that consume it.""" + + def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=None): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) + + def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["ENST00000357654"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "ENST00000357654" + assert system == "https://www.ensembl.org/index.html" + + def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["NM_000546.6"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "NM_000546.6" + assert system == "https://www.ncbi.nlm.nih.gov/refseq/" + + def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["CUSTOM_ID_1"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "CUSTOM_ID_1" + assert system == "transcript or gene identifier of unknown source" + + def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "TP53" + assert system == "https://www.mavedb.org/" + + def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) diff --git a/tests/lib/annotation/test_statement.py b/tests/lib/annotation/test_statement.py index 5c36aeecf..7612e4038 100644 --- a/tests/lib/annotation/test_statement.py +++ b/tests/lib/annotation/test_statement.py @@ -3,7 +3,7 @@ """ Tests for mavedb.lib.annotation.statement module. -This module tests statement creation functions for mapped variants, +This module tests statement creation functions for variants, focusing on functional impact statements and their classification. """ @@ -18,15 +18,13 @@ from mavedb.lib.annotation.annotate import variant_study_result from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import functional_evidence_line -from mavedb.lib.annotation.proposition import mapped_variant_to_experimental_variant_functional_impact_proposition -from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, -) +from mavedb.lib.annotation.proposition import variant_functional_impact_proposition +from mavedb.lib.annotation.statement import functional_statement @pytest.mark.unit -class TestMappedVariantFunctionalStatement: - """Unit tests for mapped variant functional statement creation.""" +class TestFunctionalStatement: + """Unit tests for functional statement creation.""" @pytest.mark.parametrize( "classification, expected_direction", @@ -36,30 +34,26 @@ class TestMappedVariantFunctionalStatement: (ExperimentalVariantFunctionalImpactClassification.INDETERMINATE, Direction.NEUTRAL), ], ) - def test_mapped_variant_to_functional_statement( + def test_functional_statement( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, classification, expected_direction, ): """Test functional statement creation with different classifications.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.functional_classification_of_variant", return_value=(MagicMock(label="Test Range"), classification), ): - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - result = mapped_variant_to_functional_statement( - mapped_variant, proposition, [evidence], score_calibration, classification - ) + proposition = variant_functional_impact_proposition(context) + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + result = functional_statement(context, proposition, [evidence], score_calibration, classification) assert isinstance(result, Statement) - assert result.description == f"Variant functional impact statement for {mapped_variant.variant.urn}." + assert result.description == f"Variant functional impact statement for {context.variant.urn}." assert result.specifiedBy assert result.contributions assert len(result.contributions) == 3 # API, VRS, and score calibration contributions @@ -67,7 +61,7 @@ def test_mapped_variant_to_functional_statement( assert result.direction == expected_direction assert result.classification.primaryCoding.code.root == classification.value assert ( - result.classification.primaryCoding.system == "ga4gh-gks-term:experimental-var-func-impact-classification" + result.classification.primaryCoding.system == "ga4gh-gkm-term:experimental-var-func-impact-classification" ) assert result.hasEvidenceLines assert len(result.hasEvidenceLines) == 1 @@ -75,24 +69,22 @@ def test_mapped_variant_to_functional_statement( def test_no_calibrations_raises_value_error( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, ): """Test that missing score calibrations raises ValueError in evidence line creation.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] # Set calibrations to None to trigger the error - mapped_variant.variant.score_set.score_calibrations = None + context.variant.score_set.score_calibrations = None - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + proposition = variant_functional_impact_proposition(context) with pytest.raises(ValueError, match="does not have a score set with score calibrations"): # Use a dummy score calibration for evidence line creation, but it should not be reached due to the error - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - mapped_variant_to_functional_statement( - mapped_variant, + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + functional_statement( + context, proposition, [evidence], score_calibration, @@ -101,8 +93,8 @@ def test_no_calibrations_raises_value_error( @pytest.mark.unit -class TestMappedVariantPathogenicityStatement: - """Unit tests for mapped variant pathogenicity statement ACMG classification mapping.""" +class TestPathogenicityStatement: + """Unit tests for pathogenicity statement ACMG classification mapping.""" def test_pathogenic_criterion_maps_to_pathogenic(self, mock_mapped_variant): """Test that pathogenic ACMG criterion (PS3) maps to PATHOGENIC classification.""" diff --git a/tests/lib/annotation/test_study_result.py b/tests/lib/annotation/test_study_result.py index 1cf8bbf63..12b3d8366 100644 --- a/tests/lib/annotation/test_study_result.py +++ b/tests/lib/annotation/test_study_result.py @@ -14,24 +14,22 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult from ga4gh.vrs.models import MolecularVariation -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri -from mavedb.lib.annotation.study_result import ( - mapped_variant_to_experimental_variant_impact_study_result, -) +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri +from mavedb.lib.annotation.study_result import variant_impact_study_result @pytest.mark.unit class TestExperimentalVariantImpactStudyResult: """Unit tests for experimental variant impact study result creation.""" - def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_mapped_variant): - """Test creation of experimental variant impact study result from mapped variant.""" - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_variant_impact_study_result(self, mock_annotation_context): + """Test creation of experimental variant impact study result from an annotation context.""" + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." - assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." + assert isinstance(result.focus, MolecularVariation) + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -39,18 +37,18 @@ def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_m # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert mapped_variant_as_iri(mock_mapped_variant) in result.reportedIn - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert measured_allele_as_iri(mock_annotation_context.measured_allele) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn - def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): - """Test that study result handles missing mapped variant (no ClinGen allele ID).""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_no_clingen_allele_id_is_filtered_properly(self, mock_annotation_context): + """Test that study result handles a measured allele with no ClinGen allele ID.""" + mock_annotation_context.measured_allele.clingen_allele_id = None + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." - assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." + assert isinstance(result.focus, MolecularVariation) + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -58,5 +56,5 @@ def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn assert len(result.reportedIn) == 1 diff --git a/tests/lib/annotation/test_util.py b/tests/lib/annotation/test_util.py deleted file mode 100644 index d44cbca3f..000000000 --- a/tests/lib/annotation/test_util.py +++ /dev/null @@ -1,900 +0,0 @@ -# ruff: noqa: E402 - -""" -Tests for mavedb.lib.annotation.util module. - -This module tests utility functions used by annotation workflows, including -variation extraction, annotation eligibility checks, and sequence feature -resolution. -""" - -from copy import deepcopy -from types import SimpleNamespace -from unittest.mock import patch - -import pytest - -pytest.importorskip("psycopg2") -pytest.importorskip("fastapi") - -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.annotation.util import ( - _can_annotate_variant_base_assumptions, - calibrations_available_for_annotation, - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, - score_calibration_may_be_used_for_annotation, - select_strongest_functional_calibration, - select_strongest_pathogenicity_calibration, - sequence_feature_for_mapped_variant, - variation_from_mapped_variant, - vrs_object_from_mapped_variant, -) -from mavedb.lib.permissions.principal import Principal -from tests.helpers.constants import ( - TEST_SEQUENCE_LOCATION_ACCESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE, - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, -) -from tests.lib.annotation.conftest import admin_principal, make_private - - -def _has_calibrations_for_annotation(*args, **kwargs) -> bool: - """Whether any calibration survived both the eligibility and visibility checks. - - ``calibrations_available_for_annotation`` returns the surviving calibrations; the cases below predate - that and assert only on whether the list was empty. - """ - return bool(calibrations_available_for_annotation(*args, **kwargs)) - - -@pytest.mark.unit -class TestVariationExtractionUnit: - @pytest.mark.parametrize( - "variation_version", - [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], - ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], - ) - def test_variation_from_mapped_variant_post_mapped_variation(self, mock_mapped_variant, variation_version): - mock_mapped_variant.post_mapped = variation_version - - result = variation_from_mapped_variant(mock_mapped_variant).model_dump() - - assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION - assert result["location"]["start"] == 5 - assert result["location"]["end"] == 6 - - def test_variation_from_mapped_variant_no_post_mapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - with pytest.raises(MappingDataDoesntExistException): - variation_from_mapped_variant(mock_mapped_variant) - - @pytest.mark.parametrize( - "allele_dict, expected_state_type, expected_state_fields", - [ - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE, - "LiteralSequenceExpression", - {"sequence": "F"}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, - "ReferenceLengthExpression", - {"length": 5, "repeatSubunitLength": 5}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - "LengthExpression", - {"length": 5}, - ), - ], - ids=["lse_state", "rle_state", "length_expression_state"], - ) - def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): - result = vrs_object_from_mapped_variant(allele_dict).model_dump() - - assert result["state"]["type"] == expected_state_type - for k, v in expected_state_fields.items(): - assert result["state"][k] == v - - def test_unknown_allele_state_type_raises_value_error(self): - allele_dict = { - **TEST_VALID_POST_MAPPED_VRS_ALLELE, - "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, - } - - with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): - vrs_object_from_mapped_variant(allele_dict) - - @pytest.mark.parametrize( - "member_dict, expected_state_type", - [ - (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), - ], - ids=["lse_members", "rle_members", "length_expression_members"], - ) - def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): - mapping_results = {"type": "CisPhasedBlock", "members": [member_dict, member_dict]} - - result = vrs_object_from_mapped_variant(mapping_results).model_dump() - - assert result["type"] == "CisPhasedBlock" - assert len(result["members"]) == 2 - assert result["members"][0]["state"]["type"] == expected_state_type - - -@pytest.mark.unit -class TestBaseAnnotationAssumptionsUnit: - def test_base_assumption_check_returns_false_when_score_is_none(self, mock_mapped_variant): - mock_mapped_variant.variant.data = {"score_data": {"score": None}} - - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is False - - def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_mapped_variant): - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is True - - -@pytest.mark.unit -class TestScoreCalibrationMayBeUsedForAnnotation: - def test_returns_false_for_research_use_only_when_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=False, - ) - is False - ) - - def test_returns_true_for_research_use_only_when_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=True, - ) - is True - ) - - def test_returns_false_when_functional_classifications_missing( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.functional_classifications = [] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False - - def test_returns_false_for_pathogenicity_without_acmg_classifications( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for functional_classification in acmg_removed: - functional_classification["acmgClassification"] = None - calibration.functional_classifications = acmg_removed - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False - - def test_returns_true_for_pathogenicity_with_any_acmg_classification( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True - - -@pytest.mark.unit -class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: - """ - Unit tests for calibration availability, via the _has_calibrations_for_annotation adapter below. - This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. - """ - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = None - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = [] - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False - - @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ): - """Test that research use only calibrations are excluded by default.""" - # Make all calibrations research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: - calibration.research_use_only = True - - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, annotation_type) - is False - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( - self, kind, variant_fixture, request - ): - """Test that research use only calibrations are included when explicitly allowed.""" - mock_mapped_variant = request.getfixturevalue(variant_fixture) - # Make all calibrations research use only - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.primary = False - calibration.research_use_only = True - - assert ( - _has_calibrations_for_annotation(mock_mapped_variant, kind, allow_research_use_only_calibrations=True) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.functional_classifications = None - - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False - - def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - for fr in acmg_classification_removed: - fr["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _has_calibrations_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is False - ) - - def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - acmg_classification_removed[0]["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _has_calibrations_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is True - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for functional annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") - is True - ) - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _has_calibrations_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - def test_score_range_check_handles_mixed_functional_classifications( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test behavior when some calibrations have functional classifications and some don't.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # First calibration has functional classifications (should already exist) - # Second calibration has no functional classifications - calibrations[1].functional_classifications = None - - # Should return True because at least one calibration has valid functional classifications - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") - is True - ) - - def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Remove ACMG classifications from all functional classifications - for calibration in calibrations: - if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: - acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for fc in acmg_classification_removed: - if "acmgClassification" in fc: - fc["acmgClassification"] = None - calibration.functional_classifications = acmg_classification_removed - - # Should return False because no ACMG classifications exist - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity") - is False - ) - - def test_functional_annotation_with_empty_functional_classifications_list( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that functional annotation fails when functional classifications list is empty.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Set functional classifications to empty list - for calibration in calibrations: - calibration.functional_classifications = [] - - assert ( - _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") - is False - ) - - -@pytest.mark.unit -class TestCalibrationAvailabilityIsScopedToTheCaller: - """A calibration's READ rule is stricter than its score set's, so publishing a score set does not - publish its calibrations. These cases exist because this function once read - ``score_set.score_calibrations`` directly and handed private calibrations to anyone. - """ - - def test_a_private_calibration_is_not_available_for_annotation( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) - - assert calibrations_available_for_annotation(mapped_variant, "functional") == [] - - def test_omitting_the_principal_withholds_rather_than_widens( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - # The whole design rests on an omitted principal meaning "the public". Passing an explicitly - # anonymous principal and passing none at all must agree. - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) - - assert calibrations_available_for_annotation( - mapped_variant, "functional" - ) == calibrations_available_for_annotation(mapped_variant, "functional", principal=Principal()) - - def test_an_entitled_caller_still_receives_a_private_calibration( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) - - assert calibrations_available_for_annotation(mapped_variant, "functional", principal=admin_principal()) != [] - - def test_a_public_calibration_is_still_available_to_anyone( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - # The counterweight: withholding private calibrations must not withhold what publishing released. - assert ( - calibrations_available_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - != [] - ) - - -@pytest.mark.unit -class TestPathogenicityAnnotationEligibilityUnit: - def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", return_value=False): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util.calibrations_available_for_annotation", - return_value=[], - ): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - assert ( - can_annotate_variant_for_pathogenicity_evidence( - mock_mapped_variant_with_pathogenicity_calibration_score_set - ) - is True - ) - - -@pytest.mark.unit -class TestFunctionalAnnotationEligibilityUnit: - def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", - return_value=False, - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_false_when_functional_classifications_check_fails( - self, mock_mapped_variant - ): - with patch( - "mavedb.lib.annotation.util.calibrations_available_for_annotation", - return_value=[], - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - assert ( - can_annotate_variant_for_functional_statement(mock_mapped_variant_with_functional_calibration_score_set) - is True - ) - - -@pytest.mark.unit -class TestSequenceFeatureForMappedVariantUnit: - def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=None): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["ENST00000357654"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "ENST00000357654" - assert system == "https://www.ensembl.org/index.html" - - def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["NM_000546.6"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "NM_000546.6" - assert system == "https://www.ncbi.nlm.nih.gov/refseq/" - - def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["CUSTOM_ID_1"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "CUSTOM_ID_1" - assert system == "transcript or gene identifier of unknown source" - - def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "TP53" - assert system == "https://www.mavedb.org/" - - def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - -@pytest.mark.integration -class TestAnnotationUtilIntegration: - """Integration tests for utility helpers using persisted DB-backed variants.""" - - def test_variation_from_persisted_mapped_variant(self, setup_lib_db_with_mapped_variant): - setup_lib_db_with_mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE - variation = variation_from_mapped_variant(setup_lib_db_with_mapped_variant) - - assert variation is not None - assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} - - def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): - # Make score presence explicit so a negative result is due to missing calibrations. - setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} - - pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence(setup_lib_db_with_mapped_variant) - functional_allowed = can_annotate_variant_for_functional_statement(setup_lib_db_with_mapped_variant) - - # DB fixture score sets do not include calibrations by default, so both should be False. - assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] - assert pathogenicity_allowed is False - assert functional_allowed is False - - -@pytest.mark.unit -class TestSelectStrongestFunctionalCalibrationUnit: - """Unit tests for select_strongest_functional_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_functional_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_returns_first_when_all_agree(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that first calibration is returned when all have same classification.""" - from copy import deepcopy - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - # Create multiple calibrations with same classification - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] # Should return the first one - - def test_defaults_to_normal_on_conflict(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that normal classification is preferred when there are conflicts.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different classifications - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - side_effect=[ - (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), - (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), - ], - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return the normal classification (second one) - assert calibration == calibration2 - assert functional_range.label == "Normal" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range but INDETERMINATE classification (variant not in any range) - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None - - -@pytest.mark.unit -class TestSelectStrongestPathogenicityCalibrationUnit: - """Unit tests for select_strongest_pathogenicity_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_pathogenicity_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_selects_strongest_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that calibration with strongest evidence is selected.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different evidence strengths - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ( - MagicMock(label="Strong"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.VERY_STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with VERY_STRONG evidence - assert calibration == calibration2 - assert functional_range.label == "Strong" - - def test_defaults_to_uncertain_on_tie(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength but different criteria (pathogenic vs benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Pathogenic"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration but None for range to indicate uncertain significance - assert calibration == calibration1 - assert functional_range is None - - def test_returns_classification_when_all_tied_are_same_type( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that classification is returned normally when all tied candidates are the same type.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength and same type of criteria (both benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Benign1"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign2"), - VariantPathogenicityEvidenceLine.Criterion.BP1, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with its range (no conflict, so normal classification) - assert calibration == calibration1 - assert functional_range.label == "Benign1" - - def test_handles_none_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that None evidence strength is handled correctly.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock with None and actual strength - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with actual strength (second) - assert calibration == calibration2 - assert functional_range.label == "Moderate" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range for all calibrations (variant not in any range) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None diff --git a/tests/lib/clingen/test_allele_registry.py b/tests/lib/clingen/test_allele_registry.py index 45a04db6d..c143dbbc1 100644 --- a/tests/lib/clingen/test_allele_registry.py +++ b/tests/lib/clingen/test_allele_registry.py @@ -9,11 +9,14 @@ import requests from mavedb.lib.clingen.allele_registry import ( + clingen_allele_url, get_associated_clinvar_allele_id, get_canonical_pa_ids, get_clingen_allele_data, get_matching_registered_ca_ids, + refresh_clingen_allele_data_expiry, ) +from mavedb.lib.clingen.cache import CACHE_TTL_SECONDS @pytest.mark.unit @@ -454,6 +457,32 @@ async def test_different_functions_share_raw_data_cache(self, mock_request, clea assert mock_request.call_count == 1 # Only one API call for both functions +@pytest.mark.unit +@mock.patch("mavedb.lib.clingen.allele_registry.requests.get") +class TestRefreshClingenAlleleDataExpiry: + @pytest.mark.asyncio + async def test_restarts_the_expiry_of_cached_data(self, mock_request, clear_cache): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"@id": "CA_TTL"} + mock_request.return_value = mock_response + await get_clingen_allele_data("CA_TTL") + + with mock.patch.object( + get_clingen_allele_data.cache, # type: ignore[attr-defined] + "expire", + wraps=get_clingen_allele_data.cache.expire, # type: ignore[attr-defined] + ) as expire: + assert await refresh_clingen_allele_data_expiry("CA_TTL") is True + + expire.assert_awaited_once_with("v1:get_clingen_allele_data:CA_TTL", CACHE_TTL_SECONDS) + + @pytest.mark.asyncio + async def test_reports_uncached_data(self, mock_request, clear_cache): + assert await refresh_clingen_allele_data_expiry("CA_NOT_CACHED") is False + mock_request.assert_not_called() + + @pytest.mark.unit @mock.patch("mavedb.lib.clingen.allele_registry.requests.get") class TestCacheBackendFailure: @@ -511,3 +540,11 @@ async def test_cache_fully_down_calls_api_every_time(self, mock_request, clear_c assert result1 == "333333" assert result2 == "333333" assert mock_request.call_count == 2 # No caching — both calls hit the API + + +@pytest.mark.unit +@pytest.mark.parametrize("allele_id", ["CA123", "PA456"]) +def test_clingen_allele_url_links_the_registry_page(allele_id): + assert clingen_allele_url(allele_id) == ( + f"https://reg.clinicalgenome.org/redmine/projects/registry/genboree_registry/by_caid?caid={allele_id}" + ) diff --git a/tests/lib/clingen/test_alleles.py b/tests/lib/clingen/test_alleles.py new file mode 100644 index 000000000..440646244 --- /dev/null +++ b/tests/lib/clingen/test_alleles.py @@ -0,0 +1,53 @@ +"""Unit tests for the pure allele-grouping primitive shared by the annotation jobs.""" + +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, group_alleles_for_annotation + + +def _row(allele_id, variant_id, *, caid="CA1"): + return ScoreSetAlleleRow( + allele_id=allele_id, + post_mapped={"type": "Allele"}, + clingen_allele_id=caid, + variant_id=variant_id, + ) + + +def test_collapses_rows_to_one_payload_per_allele(): + rows = [ + _row(1, 10), + _row(1, 11), + _row(2, 12, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups == {1: "CA1", 2: "CA2"} + + +def test_shared_allele_yields_a_single_entry(): + """An allele linked by multiple variants is deduped to one work-unit (events are allele-keyed).""" + rows = [ + _row(1, 10), + _row(1, 11), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups == {1: "CA1"} + + +def test_payload_returning_none_skips_the_allele(): + """Returning None from payload drops the allele entirely — replacing each job's ad-hoc + 'no CAID / no HGVS -> continue' filter.""" + rows = [ + _row(1, 10, caid=None), + _row(2, 11, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups == {2: "CA2"} + + +def test_empty_rows_yield_empty_grouping(): + assert group_alleles_for_annotation([], payload=lambda r: r.clingen_allele_id) == {} diff --git a/tests/lib/clingen/test_content_constructors.py b/tests/lib/clingen/test_content_constructors.py index 7aab47f76..f691a4497 100644 --- a/tests/lib/clingen/test_content_constructors.py +++ b/tests/lib/clingen/test_content_constructors.py @@ -11,7 +11,6 @@ ) from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE from mavedb import __version__ -import pytest from tests.helpers.constants import ( TEST_HGVS_IDENTIFIER, @@ -54,10 +53,8 @@ def test_construct_ldh_submission_event(): } -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None - result = construct_ldh_submission_entity(mock_variant, mapped_variant) +def test_construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele): + result = construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele) assert "MaveDBMapping" in result assert len(result["MaveDBMapping"]) == 1 @@ -65,15 +62,9 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ assert mapping["entContent"]["mavedb_id"] == VALID_VARIANT_URN assert mapping["entContent"]["score"] == 1.0 - - if has_mapped_variant: - assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" - else: - assert "pre_mapped" not in mapping["entContent"] - assert "post_mapped" not in mapping["entContent"] - assert "mapping_api_version" not in mapping["entContent"] + assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" assert mapping["entId"] == VALID_VARIANT_URN assert ( @@ -82,12 +73,10 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ ) -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None +def test_construct_ldh_submission(mock_variant, mock_mapping_record, mock_allele): variant_content = [ - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), ] uuid_1 = UUID("12345678-1234-5678-1234-567812345678") diff --git a/tests/lib/clingen/test_services.py b/tests/lib/clingen/test_services.py index f0a4f5103..316a6a1a4 100644 --- a/tests/lib/clingen/test_services.py +++ b/tests/lib/clingen/test_services.py @@ -11,7 +11,11 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") -from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD +from mavedb.lib.clingen.constants import ( + CLINGEN_HTTP_TIMEOUT, + GENBOREE_ACCOUNT_NAME, + GENBOREE_ACCOUNT_PASSWORD, +) from mavedb.lib.clingen.services import ( ClinGenAlleleRegistryService, ClinGenLdhService, @@ -61,6 +65,7 @@ def test_authenticate_with_new_jwt(self, mock_existing_jwt, mock_post, clingen_s mock_post.assert_called_once_with( f"https://genboree.org/auth/usr/gb:{GENBOREE_ACCOUNT_NAME}/auth", json={"type": "plain", "val": GENBOREE_ACCOUNT_PASSWORD}, + timeout=CLINGEN_HTTP_TIMEOUT, ) @patch("mavedb.lib.clingen.services.requests.post") @@ -150,6 +155,7 @@ def test_dispatch_submissions_success(self, mock_batched, mock_authenticate, moc url=clingen_service.url, json=submission, headers={"Authorization": "Bearer test_jwt_token", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) @patch("mavedb.lib.clingen.services.requests.put") @@ -169,6 +175,7 @@ def test_dispatch_submissions_failure(self, mock_authenticate, mock_request, cli url=clingen_service.url, json=submission, headers={"Authorization": "Bearer test_jwt_token", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) @patch("mavedb.lib.clingen.services.requests.put") @@ -211,6 +218,7 @@ def test_dispatch_submissions_no_batching(self, mock_batched, mock_authenticate, url=clingen_service.url, json=submission, headers={"Authorization": "Bearer test_jwt_token", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) diff --git a/tests/lib/clinvar/test_utils.py b/tests/lib/clinvar/test_utils.py index 082c55586..58b2d29f3 100644 --- a/tests/lib/clinvar/test_utils.py +++ b/tests/lib/clinvar/test_utils.py @@ -8,6 +8,7 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import ( + clinvar_variation_url, fetch_clinvar_variant_data, validate_clinvar_variant_summary_date, ) @@ -240,3 +241,8 @@ def mock_get(url, **kwargs): assert not stale_file.exists() pkl_files = list(tmp_path.glob("variant_summary_2020-10.parsed.*.pkl")) assert len(pkl_files) == 1 + + +@pytest.mark.unit +def test_clinvar_variation_url_links_the_variation_record(): + assert clinvar_variation_url("376654") == "https://www.ncbi.nlm.nih.gov/clinvar/variation/376654/" diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py index 2ac2183fb..a6e28a698 100644 --- a/tests/lib/conftest.py +++ b/tests/lib/conftest.py @@ -135,6 +135,9 @@ def setup_lib_db_with_variant(session, setup_lib_db_with_score_set): def setup_lib_db_with_mapped_variant(session, setup_lib_db_with_variant): """ Sets up the lib test db with a user, reference, license, and a score set. + + Seeds only the frozen ``MappedVariant`` table; tests that need a mapping on the allele substrate seed + their own record, so that they control its alleles. ``tests/lib/csv`` overrides this to add one. """ mapped_variant = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=setup_lib_db_with_variant.id) @@ -307,6 +310,23 @@ def mock_mapped_variant(mock_variant): return mv +@pytest.fixture +def mock_mapping_record(mock_mapped_variant): + # Pre-mapped data and the mapping API version now live on the per-variant MappingRecord. + rec = mock.Mock() + rec.pre_mapped = mock_mapped_variant.pre_mapped + rec.mapping_api_version = mock_mapped_variant.mapping_api_version + return rec + + +@pytest.fixture +def mock_allele(mock_mapped_variant): + # Post-mapped data now lives on the (cross-variant deduped) Allele. + allele = mock.Mock() + allele.post_mapped = mock_mapped_variant.post_mapped + return allele + + @pytest.fixture def mock_mapped_variant_with_functional_calibration_score_set( mock_mapped_variant, mock_variant_with_functional_calibration_score_set diff --git a/tests/lib/csv/conftest.py b/tests/lib/csv/conftest.py new file mode 100644 index 000000000..29230b84c --- /dev/null +++ b/tests/lib/csv/conftest.py @@ -0,0 +1,46 @@ +# ruff: noqa: E402 + +"""Fixtures scoped to the CSV library tests.""" + +import pytest + +pytest.importorskip("psycopg2") + +from tests.helpers.constants import TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record + + +@pytest.fixture +def setup_lib_db_with_mapped_variant(session, setup_lib_db_with_variant, setup_lib_db_with_mapped_variant): + """Give the shared fixture's variant a live mapping on the allele substrate as well. + + The CSV surfaces resolve their mapping-derived namespaces from the allele graph, not from the frozen + ``MappedVariant`` row the parent fixture seeds, so without this every such column comes back NA. + Overridden here rather than in the parent conftest because tests elsewhere under ``tests/lib`` seed + their own records, and a second live record per variant violates ``uq_mapping_records_current``. + + Seeded as the full g/c/p triple the CSV reconstructs from — an authoritative genomic allele, its + projection-group cdna sibling, and the protein apex — so a test can populate a non-measured level's HGVS + by setting it on the measured projection group. Mirrors ``tests.helpers.util.score_set.seed_csv_substrate``. + """ + measured = GroupSpec(hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"]) + seed_mapping_record( + session, + setup_lib_db_with_variant, + assay_level="genomic", + hgvs_assay_level=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + alleles=[ + AlleleSpec( + digest="lib-csv-genomic", + level="genomic", + is_authoritative=True, + group=measured, + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["vep_functional_consequence"], + ), + AlleleSpec(digest="lib-csv-cdna", level="cdna", group=measured), + AlleleSpec(digest="lib-csv-protein", level="protein"), + ], + ) + session.refresh(setup_lib_db_with_mapped_variant) + return setup_lib_db_with_mapped_variant diff --git a/tests/lib/csv/test_build_limiter.py b/tests/lib/csv/test_build_limiter.py new file mode 100644 index 000000000..8c69ddae5 --- /dev/null +++ b/tests/lib/csv/test_build_limiter.py @@ -0,0 +1,63 @@ +# ruff: noqa: E402 + +import threading + +import pytest + +pytest.importorskip("fastapi") + +from fastapi import HTTPException + +from mavedb.lib.csv import build_limiter +from mavedb.lib.csv.build_limiter import csv_build_slot, rows_to_build + + +@pytest.fixture +def one_slot(monkeypatch): + """A single build slot, a 1000-row threshold and no waiting, so a held slot refuses immediately.""" + slots = threading.BoundedSemaphore(1) + monkeypatch.setattr(build_limiter, "_slots", slots) + monkeypatch.setattr(build_limiter, "CSV_BUILD_MIN_ROWS", 1000) + monkeypatch.setattr(build_limiter, "CSV_BUILD_WAIT_SECONDS", 0.01) + return slots + + +@pytest.mark.unit +@pytest.mark.parametrize( + "num_variants, start, limit, expected", + [(648022, None, None, 648022), (648022, 600000, 50000, 48022), (648022, None, 5, 5), (10, 20, None, 0)], +) +def test_rows_to_build(num_variants, start, limit, expected): + assert rows_to_build(num_variants, start, limit) == expected + + +@pytest.mark.unit +class TestCsvBuildSlot: + def test_refuses_a_large_build_when_every_slot_is_held(self, one_slot): + one_slot.acquire() + + with pytest.raises(HTTPException) as exc_info: + with csv_build_slot(5000): + pass + + assert exc_info.value.status_code == 503 + assert exc_info.value.headers == {"Retry-After": str(build_limiter.RETRY_AFTER_SECONDS)} + + def test_small_builds_skip_the_cap(self, one_slot): + one_slot.acquire() + + with csv_build_slot(999): + pass + + def test_releases_the_slot_even_when_the_build_fails(self, one_slot): + with pytest.raises(RuntimeError): + with csv_build_slot(5000): + raise RuntimeError("build failed") + + assert one_slot.acquire(blocking=False) + + def test_zero_slots_disables_the_cap(self, monkeypatch): + monkeypatch.setattr(build_limiter, "_slots", None) + + with csv_build_slot(10_000_000): + pass diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index c0d97fa68..595a60343 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -16,7 +16,7 @@ from mavedb.lib.csv.namespaces import CsvNamespace from tests.helpers.constants import VALID_CALIBRATION_URN from tests.helpers.mocks.factories import create_mock_mapped_variant -from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids +from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids, to_csv_mapped_row # --------------------------------------------------------------------------- # MockVariant @@ -617,7 +617,9 @@ def test_a_row_composes_and_carries_every_planned_column(self, shape): mapped_variant = shape.build(create_mock_mapped_variant) plan = self._plan() - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) assert set(row) == set(assemble_csv_headers(plan.namespaced_columns)) @@ -634,7 +636,9 @@ def test_no_cell_leaks_a_mock(self, shape): mapped_variant = shape.build(create_mock_mapped_variant) plan = self._plan() - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) leaked = {column: value for column, value in row.items() if "Mock" in str(value)} assert not leaked, f"{shape.name} leaked mock reprs into the row: {leaked}" @@ -645,7 +649,9 @@ def test_a_mapping_without_hgvs_columns_renders_them_na(self): mapped_variant = shape.build(create_mock_mapped_variant) plan = self._plan() - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) assert row["post_mapped_hgvs_c"] == "NA" assert row["post_mapped_hgvs_at_assay_level"] == "NA" @@ -658,7 +664,9 @@ def test_the_row_serializes_to_csv(self, shape): plan = self._plan() columns = assemble_csv_headers(plan.namespaced_columns) - row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + row = variant_to_csv_row( + mapped_variant.variant, plan.namespaced_columns, mapping=to_csv_mapped_row(mapped_variant) + ) rendered = rows_to_csv([row], columns) assert len(list(csv.reader(StringIO(rendered)))) == 2 diff --git a/tests/lib/csv/test_deprecated_params.py b/tests/lib/csv/test_deprecated_params.py index fa63fb30e..6f47c2e57 100644 --- a/tests/lib/csv/test_deprecated_params.py +++ b/tests/lib/csv/test_deprecated_params.py @@ -132,6 +132,7 @@ def test_a_single_deprecation_is_announced(self): assert resolved.response_headers == { "Deprecation": "true", + "Link": '; rel="deprecation"', "Warning": '299 - "drop_na_columns is deprecated, use drop_unused_hgvs_columns"', } diff --git a/tests/lib/csv/test_fetch.py b/tests/lib/csv/test_fetch.py new file mode 100644 index 000000000..b7febd43f --- /dev/null +++ b/tests/lib/csv/test_fetch.py @@ -0,0 +1,48 @@ +# ruff: noqa: E402 + +from types import SimpleNamespace + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import text + +from mavedb.lib.csv.columns import plan_csv_columns +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from tests.helpers.util.query_plan import captured_statements + +VARIANT_NUMBER_INDEX = "ix_variants_scoreset_number" + + +def _captured_variant_query(session, namespaces, start=None, limit=None): + """Run the score set CSV fetch and return the statement and parameters it sent for the variant rows.""" + plan = plan_csv_columns({"score_columns": ["score"]}, namespaces) + with captured_statements(session) as captured: + fetch_variant_csv_data( + session, + plan.namespaced_columns, + plan.clinvar_namespaces, + score_set=SimpleNamespace(id=1), + start=start, + limit=limit, + ) + + return next((statement, parameters) for statement, parameters in captured if "FROM variants" in statement) + + +@pytest.mark.unit +class TestVariantNumberOrdering: + @pytest.mark.parametrize("namespaces", [["scores"], ["scores", "mavedb"]], ids=["scores", "with_mappings"]) + def test_score_set_query_is_served_by_the_variant_number_index(self, session, namespaces): + statement, parameters = _captured_variant_query(session, namespaces, start=250000, limit=50000) + + # Rule out the alternatives so the plan shows whether the index can serve the ORDER BY at all. + session.execute(text("SET LOCAL enable_seqscan = off")) + session.execute(text("SET LOCAL enable_sort = off")) + cursor = session.connection().connection.cursor() + cursor.execute("EXPLAIN " + statement, parameters) + plan = "\n".join(row[0] for row in cursor.fetchall()) + + assert VARIANT_NUMBER_INDEX in plan + assert "Sort" not in plan diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 37029fd18..0b67aec3d 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -2,14 +2,18 @@ import csv import io -from datetime import date -from unittest.mock import Mock, patch +from unittest.mock import Mock + +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record +from tests.helpers.util.query_plan import captured_statements +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from unittest.mock import patch import pytest pytest.importorskip("psycopg2") -from sqlalchemy import event from mavedb.lib.csv.namespaces import calibration_namespace_for_urn, is_valid_csv_namespace from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv @@ -18,15 +22,22 @@ available_variant_csv_namespaces, get_variant_csv, ) +from mavedb.lib.mondo import get_generic_disease_term from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.acmg_classification import ACMGClassification -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions from mavedb.models.enums.user_role import UserRole from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet @@ -67,6 +78,7 @@ def _add_pathogenicity_calibration(db, score_set, variants_in_abnormal_range, ur primary=True, private=False, calibration_metadata={}, + disease_term=get_generic_disease_term(db), created_by_id=score_set.created_by_id, modified_by_id=score_set.modified_by_id, ) @@ -128,6 +140,7 @@ def _add_rangeless_calibration(db, score_set, urn, title): primary=True, private=False, calibration_metadata={}, + disease_term=get_generic_disease_term(db), created_by_id=score_set.created_by_id, modified_by_id=score_set.modified_by_id, ) @@ -168,21 +181,101 @@ def _add_second_score_set_with_equivalent_variant(db, first_score_set, clingen_a db.commit() db.refresh(mapped_variant) + # Equivalence is resolved on the allele substrate: two measurements are the same variant when their + # authoritative alleles carry the same ClinGen allele id. + seed_mapping_record( + db, + variant, + assay_level="genomic", + alleles=[ + AlleleSpec( + digest=f"equivalent-{variant.id}", + level="genomic", + is_authoritative=True, + clingen_allele_id=clingen_allele_id, + ) + ], + ) + db.refresh(mapped_variant) + return score_set, variant, mapped_variant def _add_clinvar_control(db, mapped_variant, significance, review_status, db_version): - mapped_variant.clinical_controls.append( - ClinicalControl( - db_identifier="183058", - gene_symbol="PTEN", - clinical_significance=significance, - clinical_review_status=review_status, - db_name="ClinVar", - db_version=db_version, - ) + """Link a control to the variant's authoritative allele, which is where the CSV layer reads it.""" + control = ClinvarControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, ) - db.add(mapped_variant) + db.add(control) + db.commit() + + allele = db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == mapped_variant.variant_id) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).first() + assert allele is not None, "no authoritative allele to link a ClinVar control to" + + db.add(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id)) + db.commit() + + +def _live_alleles_by_level(db, variant_id): + """The live mapping record's alleles, keyed by level — what the CSV actually reads.""" + alleles = db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant_id) + .where(MappingRecord.valid_to.is_(None)) + .where(MappingRecordAllele.valid_to.is_(None)) + ).all() + return {allele.level: allele for allele in alleles} + + +def _live_mapping_record(db, variant_id): + record = db.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant_id).where(MappingRecord.valid_to.is_(None)) + ).one() + return record + + +def _measured_group(db, variant_id): + """The live record's measured projection group — where the CSV reads the non-measured levels from.""" + return db.scalars( + select(ProjectionGroup) + .join(MappingRecordAllele, MappingRecordAllele.projection_group == ProjectionGroup.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant_id) + .where(MappingRecord.valid_to.is_(None)) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).one() + + +def _authoritative_allele(db, variant_id): + allele = db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant_id) + .where(MappingRecord.valid_to.is_(None)) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).first() + assert allele is not None, f"variant {variant_id} has no authoritative allele" + return allele + + +def _link_gnomad(db, allele, gnomad_variant): + db.add(gnomad_variant) + db.commit() + db.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) db.commit() @@ -292,7 +385,8 @@ def test_research_use_only_calibration_is_offered_but_labelled(self, session, se if entry.namespace == CALIBRATION_NS_1 ) - assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.label == "Provisional Calibration" + assert entry.research_use_only is True assert entry.selected_by_default is False def test_research_use_only_calibration_is_not_in_the_default_download( @@ -354,8 +448,7 @@ def test_explicit_namespaces_restrict_columns(self, session, setup_lib_db_with_m def test_equivalent_measurements_share_clingen_allele_id(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) + _authoritative_allele(session, mapped_variant.variant_id).clingen_allele_id = "CA123456" session.commit() variant = mapped_variant.variant @@ -375,8 +468,7 @@ def test_each_measurement_is_interpreted_only_under_its_own_calibrations( ): """A score from one assay carries no meaning under another assay's thresholds.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) + _authoritative_allele(session, mapped_variant.variant_id).clingen_allele_id = "CA123456" session.commit() variant = mapped_variant.variant @@ -407,8 +499,7 @@ def test_each_measurement_is_interpreted_only_under_its_own_calibrations( def test_calibration_entries_report_their_score_set(self, session, setup_lib_db_with_mapped_variant): """A calibration means nothing against another score set's scores, so say which one owns it.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) + _authoritative_allele(session, mapped_variant.variant_id).clingen_allele_id = "CA123456" session.commit() variant = mapped_variant.variant @@ -457,20 +548,21 @@ def test_variant_without_clingen_allele_id_stands_alone(self, session, setup_lib def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" - mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" - mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" - mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X - mapped_variant.vep_functional_consequence = "missense_variant" - mapped_variant.clingen_allele_id = "CA123456" - mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) - session.add(mapped_variant) + # Every level is the measured projection group's string, including the normalized measured one. + by_level = _live_alleles_by_level(session, mapped_variant.variant_id) + record = _live_mapping_record(session, mapped_variant.variant_id) + record.hgvs_assay_level = "NC_000010.11:g.87933146_87933147delinsCT" + measured_group = _measured_group(session, mapped_variant.variant_id) + measured_group.hgvs_g = "NC_000010.11:g.87933147C>T" + measured_group.hgvs_c = "NM_000314.8:c.100A>G" + measured_group.hgvs_p = "NP_000305.3:p.Lys34Glu" + authoritative = by_level["genomic"] + authoritative.vrs_digest = TEST_GA4GH_IDENTIFIER + authoritative.clingen_allele_id = "CA123456" session.commit() + _link_gnomad(session, authoritative, GnomADVariant(**TEST_GNOMAD_VARIANT)) - # Patched where it is used, not where it is defined: `fetch` binds the value with a `from` - # import, so patching `mavedb.lib.gnomad` would leave the query filtering on the real version. - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" @@ -481,33 +573,63 @@ def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) assert rows[0]["clingen.clingen_allele_id"] == "CA123456" - def test_post_mapped_vrs_id_is_never_synthesized_from_digest(self, session, setup_lib_db_with_mapped_variant): - """A post-mapped object carrying only a ``digest`` reports NA rather than a built-up CURIE. + def test_coding_projection_without_an_allele_is_served(self, session, setup_lib_db_with_variant): + """An intronic genomic change has a coding coordinate but no cdna VRS allele: the CSV serves the measured + group's hgvs_c anyway, and no protein consequence.""" + variant = setup_lib_db_with_variant + seed_mapping_record( + session, + variant, + assay_level="genomic", + hgvs_assay_level="NC_000010.11:g.87933147C>T", + alleles=[ + AlleleSpec( + digest="intronic-genomic", + level="genomic", + is_authoritative=True, + group=GroupSpec(hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000010.11:g.87933147C>T"), + ) + ], + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" + assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000465.4:c.123+5A>G" + assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NA" + + def test_post_mapped_vrs_id_is_the_stored_identifier_verbatim(self, session, setup_lib_db_with_mapped_variant): + """The column is reported as-is, never rebuilt from the post-mapped payload. - The two stored fields are known to disagree on some rows, and only ``id`` is indexed and matched - by the VRS lookup, so a digest-derived identifier would resolve to nothing. + The old resolver parsed the stored VRS object and could fall back to synthesizing a CURIE from its + ``digest`` — which resolved to nothing, because the two fields are known to disagree on some rows + and only ``id`` is indexed. The allele substrate removes the fallback entirely: ``Allele.vrs_digest`` + already holds the full prefixed identifier, so there is nothing to derive and no payload to parse. """ mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.post_mapped = { + allele = _authoritative_allele(session, mapped_variant.variant_id) + allele.vrs_digest = TEST_GA4GH_IDENTIFIER + # A payload disagreeing with the column must not influence the reported identifier. + allele.post_mapped = { key: value for key, value in TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X.items() if key != "id" } - session.add(mapped_variant) session.commit() rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) - assert rows[0]["mavedb.post_mapped_vrs_id"] == "NA" + assert rows[0]["mavedb.post_mapped_vrs_id"] == TEST_GA4GH_IDENTIFIER + assert rows[0]["mavedb.post_mapped_vrs_id"] != TEST_GA4GH_DIGEST def test_gnomad_namespace_reports_the_whole_frequency_record(self, session, setup_lib_db_with_mapped_variant): """AF alone cannot be linked out from or judged for sampling depth; the namespace carries the record.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) - session.add(mapped_variant) - session.commit() + _link_gnomad( + session, + _authoritative_allele(session, mapped_variant.variant_id), + GnomADVariant(**TEST_GNOMAD_VARIANT), + ) - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - csv_text = get_variant_csv(session, mapped_variant.variant.urn) - rows = _parse_csv(csv_text) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) assert [column for column in rows[0].keys() if column.startswith("gnomad.")] == [ "gnomad.gnomad_af", @@ -530,26 +652,32 @@ def test_gnomad_record_absent_leaves_every_column_na(self, session, setup_lib_db """A variant with no gnomAD record reports NA across the namespace, never a zero frequency.""" mapped_variant = setup_lib_db_with_mapped_variant - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) gnomad_values = {key: value for key, value in rows[0].items() if key.startswith("gnomad.")} assert len(gnomad_values) == 7 assert set(gnomad_values.values()) == {"NA"} - def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + def test_a_link_to_an_older_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): + """The live link decides the release, not the release the deployment currently serves. + + The linker only visits alleles present in the release it ingests, so an allele absent from a newer + release keeps its prior-release link live -- indefinitely, not just during re-ingest. Filtering the + read on the served-release constant turned every such allele's frequency into a permanent NA. The + honest answer is the value we hold, labeled with the release it came from. + """ mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) - session.add(mapped_variant) - session.commit() + _link_gnomad( + session, + _authoritative_allele(session, mapped_variant.variant_id), + GnomADVariant(**{**TEST_GNOMAD_VARIANT, "db_version": "v2.1.1"}), + ) - with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", "v9.9"): - rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) - # The variant still gets a row: the version predicate is in the join's ON clause, so a gnomAD - # record from another version leaves the frequency NA rather than dropping the variant. assert len(rows) == 1 - assert rows[0]["gnomad.gnomad_af"] == "NA" + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["gnomad.gnomad_version"] == "v2.1.1" def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant @@ -568,7 +696,7 @@ def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib def test_non_clinvar_control_is_not_reported(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant mapped_variant.clinical_controls.append( - ClinicalControl( + ClinvarControl( db_identifier="ABC123", gene_symbol="BRCA1", clinical_significance="benign", @@ -623,8 +751,15 @@ def test_unmapped_variant_in_a_mapped_score_set_keeps_mapping_columns_as_na( ) session.add(mapped_sibling) session.commit() - session.add(MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_sibling.id)) - session.commit() + # The sibling carries the score set's mapping; the requested variant has none, which is what + # makes NA the honest value rather than an absent column. + seed_mapping_record( + session, + mapped_sibling, + assay_level="genomic", + hgvs_assay_level="NC_000010.11:g.1A>G", + alleles=[AlleleSpec(digest="sibling-genomic", level="genomic", is_authoritative=True)], + ) rows = _parse_csv(get_variant_csv(session, variant.urn)) @@ -643,18 +778,26 @@ def test_unmapped_variant_respects_requested_namespaces(self, session, setup_lib assert "scores.score" in csv_text.splitlines()[0] def test_superseded_mapping_is_ignored(self, session, setup_lib_db_with_mapped_variant): + """Re-mapping retires the prior record rather than clearing a flag, and only the live one is read.""" mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.current = False - session.add(mapped_variant) - session.add( - MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, - variant_id=mapped_variant.variant_id, - clingen_allele_id="CA999999", - ) - ) + _live_mapping_record(session, mapped_variant.variant_id).retire(session) session.commit() + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="genomic", + hgvs_assay_level="NC_000010.11:g.2A>G", + alleles=[ + AlleleSpec( + digest="superseding-genomic", + level="genomic", + is_authoritative=True, + clingen_allele_id="CA999999", + ) + ], + ) + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) assert len(rows) == 1 @@ -671,17 +814,9 @@ def test_does_not_load_whole_score_range_variant_collections(self, session, setu _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Second") - statements: list[str] = [] - - def record(conn, cursor, statement, parameters, context, executemany): - statements.append(statement) - - bind = session.get_bind() - event.listen(bind, "before_cursor_execute", record) - try: + with captured_statements(session) as captured: get_variant_csv(session, variant.urn) - finally: - event.remove(bind, "before_cursor_execute", record) + statements = [statement for statement, _ in captured] collection_loads = [ statement @@ -703,17 +838,9 @@ def test_namespace_discovery_does_not_scan_score_set_variants(self, session, set variant = setup_lib_db_with_mapped_variant.variant _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") - statements: list[str] = [] - - def record(conn, cursor, statement, parameters, context, executemany): - statements.append(statement) - - bind = session.get_bind() - event.listen(bind, "before_cursor_execute", record) - try: + with captured_statements(session) as captured: available_variant_csv_namespaces(session, variant.urn) - finally: - event.remove(bind, "before_cursor_execute", record) + statements = [statement for statement, _ in captured] calibration_scans = [ statement @@ -811,7 +938,8 @@ def test_research_use_only_calibration_is_offered_unchecked(self, session, setup if entry.namespace == CALIBRATION_NS_1 ) - assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.label == "Provisional Calibration" + assert entry.research_use_only is True assert entry.selected_by_default is False # Reported in its own right, not left to be inferred from the label or from the unchecked box: # this is the one reason for unchecking that decides whether the data may be published. @@ -933,79 +1061,110 @@ def test_entries_are_labeled_for_a_picker(self, session, setup_lib_db_with_mappe # --------------------------------------------------------------------------- -class TestAnchorMappingIsDeterministic: - """Everything in the CSV follows from which current mapping anchors the request.""" +class TestOneLiveMappingPerVariantIsGuaranteed: + """The CSV no longer has to tie-break between mappings claiming to be current. - def test_repeat_downloads_agree_when_several_mappings_claim_to_be_current( - self, session, setup_lib_db_with_mapped_variant - ): - """Nothing in the schema stops two rows from being current, so the pick must not be arbitrary.""" + The old substrate could not stop two ``MappedVariant`` rows from both carrying ``current=True``, so + the exports pinned a chosen row id to keep repeat downloads agreeing. The allele graph promotes that + invariant to the database -- ``uq_mapping_records_current`` is unique on ``variant_id`` where + ``valid_to IS NULL`` -- so the ambiguity those tests guarded against is now unrepresentable, and the + pinning they justified has been removed. What is worth asserting is the guarantee itself. + """ + + def test_a_second_live_mapping_record_is_rejected(self, session, setup_lib_db_with_mapped_variant): variant = setup_lib_db_with_mapped_variant.variant + # The fixture already seeded one live record for this variant. + session.add( + MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", + mapping_api_version="test.0.0", + ) + ) - newer = MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2030, 1, 1), "clingen_allele_id": "CA_NEWER"}, - variant_id=variant.id, + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + @staticmethod + def _other_score_set(session, score_set): + other = ScoreSet( + **{k: v for k, v in TEST_SEQ_SCORESET.items() if k != "target_genes"}, + urn="urn:mavedb:00000001-a-3", + experiment_id=score_set.experiment_id, + licence_id=score_set.licence_id, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, ) - session.add(newer) + session.add(other) session.commit() + return other - first = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) - second = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) - - assert first == second - # The requested variant anchors the export and comes first, so this pins which mapping was - # picked rather than merely whether the newer one appears anywhere in the output. - assert first[0]["clingen.clingen_allele_id"] == "CA_NEWER" - - def test_a_variant_with_two_current_mappings_is_reported_once(self, session, setup_lib_db_with_mapped_variant): - """Two current mappings on one variant are the same measurement twice, not two equivalents. + def test_a_record_cannot_claim_another_score_set(self, session, setup_lib_db_with_mapped_variant): + """``scoreset_id`` is carried for row-level security, so it must match the variant's score set. - Emitting both would also break the row ordering downstream, which restores the caller's order from - the variant ids alone. + The other score set exists, so the rejection comes from the composite key on + ``(variant_id, scoreset_id)``, not from a missing parent. """ - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) - session.commit() - - variant = mapped_variant.variant + variant = setup_lib_db_with_mapped_variant.variant + other = self._other_score_set(session, variant.score_set) + live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).one() + live.retire(session) session.add( - MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + MappingRecord( variant_id=variant.id, + score_set_id=other.id, + assay_level="genomic", + mapping_api_version="test.1.0", ) ) - session.commit() - # A genuine equivalent in another score set, so the widening is doing something to dedupe within. - _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + with pytest.raises(IntegrityError): + session.commit() + session.rollback() - rows = _parse_csv(get_variant_csv(session, variant.urn)) + def test_an_allele_link_cannot_claim_another_score_set(self, session, setup_lib_db_with_mapped_variant): + """A link's ``scoreset_id`` must match its mapping record's, by the composite key on the link.""" + variant = setup_lib_db_with_mapped_variant.variant + other = self._other_score_set(session, variant.score_set) + live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).one() + allele = Allele(vrs_digest="lib-csv-unlinked", level="genomic") + session.add(allele) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=live.id, score_set_id=other.id, allele_id=allele.id)) - assert [row["accession"] for row in rows] == [variant.urn, "urn:mavedb:00000001-a-2#1"] + with pytest.raises(IntegrityError): + session.commit() + session.rollback() - def test_an_equivalent_variants_extra_current_mapping_is_reported_once( - self, session, setup_lib_db_with_mapped_variant - ): - """The same rule applies to the widened rows, not just to the anchor.""" - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = "CA123456" - session.add(mapped_variant) - session.commit() + def test_a_superseded_record_does_not_block_a_new_live_one(self, session, setup_lib_db_with_mapped_variant): + """Supersession is what makes re-mapping possible: the prior record closes, the new one opens.""" + variant = setup_lib_db_with_mapped_variant.variant + live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).one() - variant = mapped_variant.variant - _, other_variant, _ = _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + live.retire(session) session.add( - MappedVariant( - **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, - variant_id=other_variant.id, + MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", + mapping_api_version="test.1.0", ) ) session.commit() - rows = _parse_csv(get_variant_csv(session, variant.urn)) - - assert [row["accession"] for row in rows] == [variant.urn, other_variant.urn] + still_live = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.valid_to.is_(None)) + ).all() + assert len(still_live) == 1 + assert still_live[0].mapping_api_version == "test.1.0" # --------------------------------------------------------------------------- @@ -1013,6 +1172,75 @@ def test_an_equivalent_variants_extra_current_mapping_is_reported_once( # --------------------------------------------------------------------------- +@pytest.mark.integration +class TestGnomadReleasesCoexistOnTheReadPath: + """A score set can hold live links at more than one gnomAD release, and the read path must serve both. + + gnomAD shadows like VEP rather than accumulating like ClinVar: the link is allele-keyed and + single-live, so a release bump supersedes an allele's prior link. But ingestion only visits alleles + present in the release being ingested, so alleles absent from a newer one keep their older link live. + A corpus therefore sits at mixed releases as its steady state, and one export has to carry both. + """ + + def _variant_at(self, session, score_set, urn_suffix, digest, db_version, allele_frequency): + variant = Variant(**{**TEST_MINIMAL_VARIANT, "urn": f"{score_set.urn}#{urn_suffix}"}, score_set_id=score_set.id) + session.add(variant) + session.commit() + seed_mapping_record( + session, + variant, + assay_level="genomic", + hgvs_assay_level=f"NC_000010.11:g.{urn_suffix}A>G", + alleles=[AlleleSpec(digest=digest, level="genomic", is_authoritative=True)], + ) + _link_gnomad( + session, + _authoritative_allele(session, variant.id), + GnomADVariant( + **{ + **TEST_GNOMAD_VARIANT, + "db_version": db_version, + "db_identifier": f"10-{urn_suffix}-A-G", + "allele_frequency": allele_frequency, + } + ), + ) + return variant + + def test_two_releases_are_reported_side_by_side_each_with_its_own_version(self, session, setup_lib_db_with_variant): + score_set = setup_lib_db_with_variant.score_set + newer = self._variant_at(session, score_set, 2, "gnomad-newer", "v4.1", 0.25) + older = self._variant_at(session, score_set, 3, "gnomad-older", "v2.1.1", 0.5) + + rows = _parse_csv(get_score_set_variants_as_csv(session, score_set, ["scores", "gnomad"], namespaced=True)) + + by_urn = {row["accession"]: row for row in rows} + # Both frequencies are served, each labeled with the release it actually came from. + assert by_urn[newer.urn]["gnomad.gnomad_af"] == "0.25" + assert by_urn[newer.urn]["gnomad.gnomad_version"] == "v4.1" + assert by_urn[older.urn]["gnomad.gnomad_af"] == "0.5" + assert by_urn[older.urn]["gnomad.gnomad_version"] == "v2.1.1" + # The variant with no gnomAD link at all is still a row, with the namespace NA. + assert by_urn[setup_lib_db_with_variant.urn]["gnomad.gnomad_af"] == "NA" + assert by_urn[setup_lib_db_with_variant.urn]["gnomad.gnomad_version"] == "NA" + + def test_neither_release_is_dropped_by_the_served_release_constant(self, session, setup_lib_db_with_variant): + """Whatever `GNOMAD_DATA_VERSION` the deployment serves, the read path does not consult it. + + Ingestion still needs the constant -- it names the release being ingested and derives the source + table -- so this asserts only that reads go off live rows. + """ + score_set = setup_lib_db_with_variant.score_set + self._variant_at(session, score_set, 2, "gnomad-a", "v4.1", 0.25) + self._variant_at(session, score_set, 3, "gnomad-b", "v2.1.1", 0.5) + + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v99.99"): + rows = _parse_csv(get_score_set_variants_as_csv(session, score_set, ["gnomad"], namespaced=True)) + + reported = {row["gnomad.gnomad_version"] for row in rows} + assert reported == {"v4.1", "v2.1.1", "NA"} + + class TestScoreSetCsvCalibrationColumns: """The score-set CSV must fill the calibration columns its discovery advertises.""" diff --git a/tests/lib/middleware/test_errors.py b/tests/lib/middleware/test_errors.py index 76a14b0e2..698b114b8 100644 --- a/tests/lib/middleware/test_errors.py +++ b/tests/lib/middleware/test_errors.py @@ -15,6 +15,8 @@ pytest.importorskip("fastapi") from fastapi.testclient import TestClient +from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError from mavedb.server_main import app @@ -41,6 +43,34 @@ def boom(): app.router.routes.remove(route) +class FakePostgresError(Exception): + """Stands in for a psycopg2 error, which carries the server's SQLSTATE as ``pgcode``.""" + + def __init__(self, pgcode): + super().__init__(pgcode) + self.pgcode = pgcode + + +def postgres_error(pgcode): + return OperationalError("SELECT 1", {}, FakePostgresError(pgcode)) + + +@pytest.fixture +def database_error_route(): + """Register a route that raises whichever error the test sets on ``raised["error"]``.""" + raised = {} + + @app.get(BOOM_PATH) + def boom(): + raise raised["error"] + + route = app.router.routes[-1] + try: + yield raised + finally: + app.router.routes.remove(route) + + @pytest.fixture def slack_error(): with patch("mavedb.lib.middleware.errors.send_slack_error") as mocked: @@ -85,3 +115,38 @@ def test_successful_request_is_untouched(self, slack_error): assert response.status_code == 200 assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) slack_error.assert_not_called() + + +@pytest.mark.unit +class TestDatabaseTimeouts: + @pytest.mark.parametrize( + "error", + [postgres_error("57014"), postgres_error("55P03"), PoolTimeoutError("QueuePool limit reached")], + ids=["statement_timeout", "lock_timeout", "pool_timeout"], + ) + def test_timeout_returns_503_with_cors_headers(self, database_error_route, slack_error, error): + database_error_route["error"] = error + + with TestClient(app) as tc: + response = tc.get(BOOM_PATH, headers={"Origin": TEST_ORIGIN}) + + assert response.status_code == 503 + assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) + assert response.json()["correlation_id"] + + def test_timeout_does_not_alert_slack(self, database_error_route, slack_error): + database_error_route["error"] = postgres_error("57014") + + with TestClient(app) as tc: + tc.get(BOOM_PATH) + + slack_error.assert_not_called() + + def test_other_database_errors_are_still_500(self, database_error_route, slack_error): + database_error_route["error"] = postgres_error("23505") + + with TestClient(app) as tc: + response = tc.get(BOOM_PATH) + + assert response.status_code == 500 + slack_error.assert_called_once() diff --git a/tests/lib/mondo/__init__.py b/tests/lib/mondo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/mondo/network/__init__.py b/tests/lib/mondo/network/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/mondo/network/test_mondo.py b/tests/lib/mondo/network/test_mondo.py new file mode 100644 index 000000000..8e056927f --- /dev/null +++ b/tests/lib/mondo/network/test_mondo.py @@ -0,0 +1,55 @@ +# ruff: noqa: E402 +"""Network tests for the MONDO/OLS integration. Require a live connection to www.ebi.ac.uk (OLS4). + +The unit tests in ``tests/lib/mondo/test_mondo.py`` mock OLS, so these are the only exposure to real upstream +behavior — a canary for changes to OLS4's endpoints, response shape, or term identity. Sockets are +blocked for unmarked tests, so these run only under the ``network`` marker and are not part of the +default suite. +""" + +import pytest + +# starlette is required for logging context functionality pulled in by mavedb.lib.mondo_ols. +pytest.importorskip("starlette") + +from mavedb.lib.mondo import MONDO_GENERIC_CODE +from mavedb.lib.mondo_ols import fetch_mondo_term, search_mondo + +# Brugada syndrome: a stable, long-established MONDO disease term used as a known-good fixture. +BRUGADA_CODE = "MONDO:0015263" + + +@pytest.mark.network +@pytest.mark.asyncio +class TestMondoOlsNetwork: + """Canary tests exercising the real OLS4 service.""" + + async def test_search_returns_known_term_with_expected_shape(self): + # /select endpoint: a known query surfaces the expected code, in the {code, label, iri} shape. + results = await search_mondo("brugada syndrome") + assert results, "OLS returned no results for a known disease query." + assert BRUGADA_CODE in {result["code"] for result in results} + first = results[0] + assert first["code"].startswith("MONDO:") + assert first["label"] + assert first["iri"].startswith("http") + + async def test_search_broad_prefix_returns_results(self): + # A two-letter prefix must still return via /select — /search timed out on these (the #754 switch). + assert await search_mondo("ca") + + async def test_fetch_known_code_returns_canonical_term(self): + # /search exact-obo_id lookup: a known code resolves to its canonical label. + resolved = await fetch_mondo_term(BRUGADA_CODE) + assert resolved is not None + assert resolved["code"] == BRUGADA_CODE + assert "brugada" in resolved["label"].lower() + + async def test_fetch_generic_root_resolves(self): + resolved = await fetch_mondo_term(MONDO_GENERIC_CODE) + assert resolved is not None + assert resolved["code"] == MONDO_GENERIC_CODE + + async def test_fetch_unknown_code_returns_none(self): + # The not-found path must stay a clean None so validation can reject unknown codes. + assert await fetch_mondo_term("MONDO:9999999") is None diff --git a/tests/lib/mondo/test_mondo.py b/tests/lib/mondo/test_mondo.py new file mode 100644 index 000000000..409fe8102 --- /dev/null +++ b/tests/lib/mondo/test_mondo.py @@ -0,0 +1,214 @@ +"""Tests for mavedb.lib.mondo and mavedb.lib.mondo_ols: MONDO term resolution, search, and +MappableConcept serialization.""" + +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +import httpx + +from mavedb.lib.exceptions import MondoServiceError +from mavedb.lib.mondo import ( + MONDO_GENERIC_CODE, + MONDO_GENERIC_LABEL, + MONDO_SYSTEM, + generic_disease_mappable_concept, + get_generic_disease_term, + mondo_iri, + mondo_suggestion_to_mappable_concept, + mondo_term_to_mappable_concept, +) +from mavedb.lib.mondo_ols import fetch_mondo_term, find_or_create_mondo_term, search_mondo +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.mondo_term import MondoTerm + + +class _FakeResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + +class _FakeAsyncClient: + """Minimal stand-in for httpx.AsyncClient returning a canned OLS payload.""" + + def __init__(self, payload): + self._payload = payload + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url, params=None): + return _FakeResponse(self._payload) + + +class _FailingAsyncClient: + """Stand-in for httpx.AsyncClient whose request fails, as when OLS is unreachable.""" + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url, params=None): + raise httpx.ConnectError("ols down") + + +def _ols_payload(docs): + return {"response": {"docs": docs}} + + +@pytest.mark.unit +class TestMondoSerialization: + def test_mondo_iri_replaces_curie_colon(self): + assert mondo_iri("MONDO:0015263") == "https://purl.obolibrary.org/obo/MONDO_0015263" + + def test_term_to_mappable_concept(self): + term = MondoTerm( + code="MONDO:0015263", system=MONDO_SYSTEM, system_version="2026-01-01", label="Brugada syndrome" + ) + concept = mondo_term_to_mappable_concept(term) + assert concept.conceptType == "Disease" + assert concept.name == "Brugada syndrome" + assert concept.primaryCoding.code.root == "MONDO:0015263" + assert concept.primaryCoding.system == MONDO_SYSTEM + assert concept.primaryCoding.iris[0].root == "https://purl.obolibrary.org/obo/MONDO_0015263" + + def test_generic_concept_is_the_mondo_root(self): + concept = generic_disease_mappable_concept() + assert concept.primaryCoding.code.root == MONDO_GENERIC_CODE + assert concept.name == MONDO_GENERIC_LABEL + + def test_suggestion_to_mappable_concept(self): + concept = mondo_suggestion_to_mappable_concept( + {"code": "MONDO:0015263", "label": "Brugada syndrome", "iri": "https://example.org/MONDO_0015263"} + ) + assert concept.primaryCoding.code.root == "MONDO:0015263" + assert concept.primaryCoding.iris[0].root == "https://example.org/MONDO_0015263" + + +@pytest.mark.asyncio +class TestMondoOlsSearch: + async def test_search_mondo_parses_ols_docs(self, monkeypatch): + payload = _ols_payload( + [ + { + "obo_id": "MONDO:0015263", + "label": "Brugada syndrome", + "iri": "https://purl.obolibrary.org/obo/MONDO_0015263", + }, + {"obo_id": "MONDO:0000001", "label": "disease or disorder"}, # missing iri → derived + ] + ) + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FakeAsyncClient(payload)) + + results = await search_mondo("brugada") + assert results[0] == { + "code": "MONDO:0015263", + "label": "Brugada syndrome", + "iri": "https://purl.obolibrary.org/obo/MONDO_0015263", + } + assert results[1]["iri"] == mondo_iri("MONDO:0000001") + + async def test_search_mondo_empty_query_skips_request(self, monkeypatch): + # No network expected; if a request were made the fake would still be needed, so assert on the shortcut. + assert await search_mondo(" ") == [] + + async def test_fetch_mondo_term_returns_none_when_absent(self, monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FakeAsyncClient(_ols_payload([]))) + assert await fetch_mondo_term("MONDO:9999999") is None + + async def test_search_mondo_wraps_http_errors_in_service_error(self, monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FailingAsyncClient()) + + with pytest.raises(MondoServiceError): + await search_mondo("brugada") + + async def test_fetch_mondo_term_wraps_http_errors_in_service_error(self, monkeypatch): + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **k: _FailingAsyncClient()) + + with pytest.raises(MondoServiceError): + await fetch_mondo_term("MONDO:0015263") + + +@pytest.mark.asyncio +class TestFindOrCreateMondoTerm: + async def test_creates_from_ols_and_is_idempotent(self, session, monkeypatch): + async def fake_fetch(code): + return {"code": code, "label": "Brugada syndrome", "iri": mondo_iri(code)} + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + term = await find_or_create_mondo_term(session, "MONDO:0015263") + assert term.code == "MONDO:0015263" + assert term.label == "Brugada syndrome" + assert term.system == MONDO_SYSTEM + + # A second call reuses the row rather than creating a duplicate. + again = await find_or_create_mondo_term(session, "MONDO:0015263") + assert again.id == term.id + assert session.query(MondoTerm).filter(MondoTerm.code == "MONDO:0015263").count() == 1 + + async def test_non_canonical_code_reuses_existing_canonical_row(self, session, monkeypatch): + # OLS normalizes both spellings to the same canonical code; the lookup must key on the resolved + # code so a second submission reuses the row instead of tripping UNIQUE(system, code). + async def fake_fetch(code): + return {"code": "MONDO:0015263", "label": "Brugada syndrome", "iri": mondo_iri("MONDO:0015263")} + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + canonical = await find_or_create_mondo_term(session, "MONDO:0015263") + # A differently-spelled submission (here lowercase) resolves to the same canonical term. + aliased = await find_or_create_mondo_term(session, "mondo:0015263") + + assert aliased.id == canonical.id + assert aliased.code == "MONDO:0015263" + assert session.query(MondoTerm).filter(MondoTerm.code == "MONDO:0015263").count() == 1 + + async def test_generic_code_resolves_without_ols(self, session, monkeypatch): + async def fail_fetch(code): + raise AssertionError("OLS should not be consulted for the generic term") + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fail_fetch) + term = await find_or_create_mondo_term(session, MONDO_GENERIC_CODE) + assert term.code == MONDO_GENERIC_CODE + assert term.label == MONDO_GENERIC_LABEL + + async def test_rejects_unknown_code(self, session, monkeypatch): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + with pytest.raises(ValidationError): + await find_or_create_mondo_term(session, "MONDO:9999999") + + async def test_ols_outage_propagates_service_error(self, session, monkeypatch): + # A controlled term is never persisted unvalidated: an OLS outage surfaces as MondoServiceError. + async def fake_fetch(code): + raise MondoServiceError("ols down") + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + with pytest.raises(MondoServiceError): + await find_or_create_mondo_term(session, "MONDO:0015263") + + +@pytest.mark.unit +class TestGenericDiseaseTerm: + def test_get_generic_disease_term_is_idempotent(self, session): + first = get_generic_disease_term(session) + assert first.code == MONDO_GENERIC_CODE + assert first.system == MONDO_SYSTEM + second = get_generic_disease_term(session) + assert second.id == first.id + assert session.query(MondoTerm).filter(MondoTerm.code == MONDO_GENERIC_CODE).count() == 1 diff --git a/tests/lib/permissions/test_score_calibration.py b/tests/lib/permissions/test_score_calibration.py index 9689ef42c..75120c6fa 100644 --- a/tests/lib/permissions/test_score_calibration.py +++ b/tests/lib/permissions/test_score_calibration.py @@ -178,7 +178,9 @@ class TestScoreCalibrationReadActionHandler: "ScoreCalibration", "private", "anonymous", Action.READ, False, 404, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_read_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_read_action helper function directly.""" @@ -305,7 +307,9 @@ class TestScoreCalibrationUpdateActionHandler: "ScoreCalibration", "published", "anonymous", Action.UPDATE, False, 401, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_update_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_update_action helper function directly.""" @@ -390,7 +394,9 @@ class TestScoreCalibrationDeleteActionHandler: "ScoreCalibration", "published", "anonymous", Action.DELETE, False, 401, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_delete_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_delete_action helper function directly.""" @@ -471,7 +477,9 @@ class TestScoreCalibrationPublishActionHandler: "ScoreCalibration", "published", "anonymous", Action.PUBLISH, False, 401, investigator_provided=False ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_publish_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_publish_action helper function directly.""" @@ -601,7 +609,9 @@ class TestScoreCalibrationChangeRankActionHandler: investigator_provided=False, ), ], - ids=lambda tc: f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}", + ids=lambda tc: ( + f"{tc.user_type}_{tc.entity_state}_{'investigator' if tc.investigator_provided else 'community'}_{tc.action.value}_{'permitted' if tc.should_be_permitted else 'denied'}" + ), ) def test_handle_change_rank_action(self, test_case: PermissionTest, entity_helper: EntityTestHelper) -> None: """Test _handle_change_rank_action helper function directly.""" diff --git a/tests/lib/permissions/test_score_set.py b/tests/lib/permissions/test_score_set.py index 7198fba97..2d5a0c497 100644 --- a/tests/lib/permissions/test_score_set.py +++ b/tests/lib/permissions/test_score_set.py @@ -245,12 +245,12 @@ class TestScoreSetSetScoresActionHandler: # Admins can set scores on any ScoreSet PermissionTest("ScoreSet", "private", "admin", Action.SET_SCORES, True), PermissionTest("ScoreSet", "published", "admin", Action.SET_SCORES, True), - # Owners can set scores on any ScoreSet they own + # Owners can only set scores on ScoreSets they own that are still unpublished PermissionTest("ScoreSet", "private", "owner", Action.SET_SCORES, True), - PermissionTest("ScoreSet", "published", "owner", Action.SET_SCORES, True), - # Contributors can set scores on any ScoreSet they contribute to + PermissionTest("ScoreSet", "published", "owner", Action.SET_SCORES, False, 403), + # Contributors are likewise limited to unpublished ScoreSets PermissionTest("ScoreSet", "private", "contributor", Action.SET_SCORES, True), - PermissionTest("ScoreSet", "published", "contributor", Action.SET_SCORES, True), + PermissionTest("ScoreSet", "published", "contributor", Action.SET_SCORES, False, 403), # Mappers cannot set scores on ScoreSets PermissionTest("ScoreSet", "private", "mapper", Action.SET_SCORES, False, 404), PermissionTest("ScoreSet", "published", "mapper", Action.SET_SCORES, False, 403), diff --git a/tests/lib/test_allele_annotations.py b/tests/lib/test_allele_annotations.py new file mode 100644 index 000000000..cedb4267c --- /dev/null +++ b/tests/lib/test_allele_annotations.py @@ -0,0 +1,162 @@ +# ruff: noqa: E402 +"""Integration tests for the digest-keyed annotation assembler (``lib/allele_annotations.py``). + +These pin: the map is keyed by ``vrs_digest`` and covers every allele handed in (an allele with no +annotations still gets an empty entry); VEP/gnomAD resolve to one value each while ClinVar is a list +(multi-live, one per release); absence is ``None``/empty; and ``as_of`` reconstructs each source at a +past instant over the immutable, content-addressed alleles. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.allele_annotations import get_allele_annotations +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="cdna"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _vep(session, allele, value, *, source_version="116", valid_from=None): + row = VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=value, + source_version=source_version, + access_date="2026-01-01", + valid_from=valid_from, + ) + session.add(row) + session.commit() + return row + + +def _gnomad(session, allele, *, allele_frequency=0.0123, db_version="4.1.0"): + gv = GnomADVariant( + db_name="gnomAD", + db_identifier=f"gnomad-{allele.vrs_digest}", + db_version=db_version, + allele_count=42, + allele_number=10000, + allele_frequency=allele_frequency, + faf95_max=0.05, + ) + session.add(gv) + session.commit() + link = GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gv.id) + session.add(link) + session.commit() + return gv + + +def _clinvar(session, allele, *, significance="Likely benign", db_version="11_2024", variation_id="12345"): + control = ClinvarControl( + db_identifier=f"cv-{db_version}-{allele.vrs_digest}", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status="criteria provided, multiple submitters, no conflicts", + db_name="ClinVar", + db_version=db_version, + clinvar_variation_id=variation_id, + ) + session.add(control) + session.commit() + link = ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id) + session.add(link) + session.commit() + return control + + +@pytest.mark.integration +def test_empty_allele_list_returns_empty_map(session, setup_lib_db): + assert get_allele_annotations(session, []) == {} + + +@pytest.mark.integration +def test_allele_without_annotations_gets_an_empty_entry(session, setup_lib_db): + """The map's keys mirror the alleles handed in — a fully un-annotated allele still gets an + (empty) block, so the envelope can join every Cat-VRS member to something.""" + allele = _allele(session, "bare-digest") + + annotations = get_allele_annotations(session, [allele]) + + assert set(annotations) == {"bare-digest"} + block = annotations["bare-digest"] + assert block.vep is None + assert block.gnomad is None + assert block.clinvar == [] + + +@pytest.mark.integration +def test_all_three_sources_present(session, setup_lib_db): + allele = _allele(session, "rich-digest") + _vep(session, allele, "missense_variant") + _gnomad(session, allele, allele_frequency=0.0123) + _clinvar(session, allele, significance="Likely benign") + + block = get_allele_annotations(session, [allele])["rich-digest"] + + assert block.vep is not None and block.vep.consequence == "missense_variant" + assert block.vep.source_version == "116" + assert block.gnomad is not None and block.gnomad.allele_frequency == 0.0123 + assert block.gnomad.faf95_max == 0.05 + assert [c.clinical_significance for c in block.clinvar] == ["Likely benign"] + + +@pytest.mark.integration +def test_clinvar_is_multi_live_across_releases(session, setup_lib_db): + """ClinVar stacks one live link per release, so an allele can carry several assertions at once.""" + allele = _allele(session, "cv-digest") + _clinvar(session, allele, significance="Likely benign", db_version="11_2024") + _clinvar(session, allele, significance="Pathogenic", db_version="05_2025") + + block = get_allele_annotations(session, [allele])["cv-digest"] + + assert {(c.db_version, c.clinical_significance) for c in block.clinvar} == { + ("11_2024", "Likely benign"), + ("05_2025", "Pathogenic"), + } + + +@pytest.mark.integration +def test_map_is_keyed_by_digest_across_multiple_alleles(session, setup_lib_db): + a1 = _allele(session, "digest-1", level="cdna") + a2 = _allele(session, "digest-2", level="protein") + _vep(session, a1, "missense_variant") + _gnomad(session, a2) + + annotations = get_allele_annotations(session, [a1, a2]) + + assert set(annotations) == {"digest-1", "digest-2"} + assert annotations["digest-1"].vep is not None and annotations["digest-1"].gnomad is None + assert annotations["digest-2"].gnomad is not None and annotations["digest-2"].vep is None + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_annotation(session, setup_lib_db): + """A changed VEP consequence supersedes the old row; as_of selects the value live at the instant.""" + allele = _allele(session, "vep-digest") + old = _vep(session, allele, "missense_variant", source_version="110", valid_from=T0) + old.retire(session, at=T1) + session.commit() + _vep(session, allele, "stop_gained", source_version="116", valid_from=T1) + + current = get_allele_annotations(session, [allele])["vep-digest"] + historical = get_allele_annotations(session, [allele], as_of=T1 - timedelta(days=1))["vep-digest"] + + assert current.vep is not None and current.vep.consequence == "stop_gained" + assert historical.vep is not None and historical.vep.consequence == "missense_variant" diff --git a/tests/lib/test_allele_detail.py b/tests/lib/test_allele_detail.py new file mode 100644 index 000000000..9402d400a --- /dev/null +++ b/tests/lib/test_allele_detail.py @@ -0,0 +1,206 @@ +# ruff: noqa: E402 +"""Integration tests for the allele-detail assembly backing ``GET /alleles/{digest}`` (+ CAID). + +``get_allele_detail`` is the allele-grain sibling of ``get_variant_detail``: it anchors on an allele +(a digest, or a CAID's genomic+coding pair) and serves the **full cross-layer equivalence class** +(``get_allele_translations``), with each member labelled *relative to the focus* — faithful +(``projection``) vs. convergent cousin vs. reverse-translation ``candidate`` — plus the digest-keyed +annotations. These tests pin the labelling in both directions (nt focus / protein focus), the full-class +membership incl. cousins, the CAID focus set, the orphan fallback, and ``as_of``. +""" + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.models.allele import Allele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest) -> Allele: + allele = session.scalar(select(Allele).where(Allele.vrs_digest == digest)) + assert allele is not None, f"allele {digest!r} not seeded" + return allele + + +def _coding_measured_specs(): + """A coding-measured record: authoritative cdna (+VEP+ClinGen), its genomic projection partner + (shared measured group), the protein apex, and a synonymous cousin (its own group).""" + measured = GroupSpec( + hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p="NP_000537.3:p.Ala406Thr" + ) + return [ + AlleleSpec( + digest="cdna-d", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + group=measured, + ), + AlleleSpec( + digest="gen-d", + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + group=measured, + ), + AlleleSpec(digest="prot-d", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr", post_mapped=_post_mapped()), + AlleleSpec( + digest="cousin-d", + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + group=GroupSpec(hgvs_c="NM_000546.6:c.1218C>T"), + ), + ] + + +@pytest.mark.integration +def test_nt_focus_labels_the_full_class_relative_to_the_queried_allele(session, setup_lib_db_with_score_set): + """Fetching the measured coding allele: the whole class comes back, each member labelled relative to + it — its genomic is a faithful coordinate projection, the protein its consequence, the cousin convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert detail.digest == "cdna-d" + assert detail.level == "cdna" + assert detail.vrs is not None and detail.vrs["type"] == "Allele" + assert set(detail.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} # full class, cousin included + + focus = detail.alleles["cdna-d"] + assert focus.is_focus is True and focus.relation is None and focus.derivation is None + assert focus.projection_of == "gen-d" + + genomic = detail.alleles["gen-d"] + assert genomic.is_focus is False + assert genomic.relation == "coordinate_representation_of" + assert genomic.derivation == "projection" + assert genomic.projection_of == "cdna-d" + + protein = detail.alleles["prot-d"] + assert protein.relation == "translation_of" + assert protein.derivation == "projection" + + cousin = detail.alleles["cousin-d"] + assert cousin.relation == "co_encodes" + assert cousin.derivation == "convergent" # distinct change sharing the consequence, not the coordinate partner + + +@pytest.mark.integration +def test_protein_focus_labels_every_nucleotide_as_a_candidate(session, setup_lib_db_with_score_set): + """Fetching the protein apex: walking down is ambiguous, so every nt member is a reverse-translation + candidate (the 'less power without a measurement' case, still labelled precisely by direction).""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "prot-d"), focus_digests={"prot-d"}) + + assert detail.alleles["prot-d"].is_focus is True + for nt in ("cdna-d", "gen-d", "cousin-d"): + assert detail.alleles[nt].relation == "encodes", nt + assert detail.alleles[nt].derivation == "candidate", nt + + +@pytest.mark.integration +def test_annotations_join_by_digest(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert set(detail.annotations) == set(detail.alleles) + assert detail.annotations["cdna-d"].vep is not None + assert detail.annotations["cdna-d"].vep.consequence == "missense_variant" + assert detail.annotations["gen-d"].vep is None # a member with no annotations still gets an empty block + + +@pytest.mark.integration +def test_caid_focus_flags_both_frames(session, setup_lib_db_with_score_set): + """A CAID names the nt change in both frames: fetched by CAID, the genomic + coding are both isFocus + (each other's coordinate partner), and only the true cousin is convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + # The CAID resolves to the {cdna, genomic} pair (both carry clingen_allele_id="CA1"). + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d", "gen-d"}) + + assert detail.alleles["cdna-d"].is_focus is True + assert detail.alleles["gen-d"].is_focus is True + assert detail.alleles["prot-d"].relation == "translation_of" + assert detail.alleles["cousin-d"].derivation == "convergent" + + +@pytest.mark.integration +def test_orphan_allele_falls_back_to_itself(session, setup_lib_db_with_score_set): + orphan = Allele(vrs_digest="orphan-d", level="cdna", hgvs_c="NM:c.9A>G", post_mapped=_post_mapped()) + session.add(orphan) + session.commit() + + detail = get_allele_detail(session, orphan, focus_digests={"orphan-d"}) + + assert set(detail.alleles) == {"orphan-d"} + assert detail.alleles["orphan-d"].is_focus is True + assert set(detail.annotations) == {"orphan-d"} + + +@pytest.mark.integration +def test_as_of_reconstructs_membership(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna", valid_from=T1) + + anchor = _allele(session, "cdna-d") + + before = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T0) + assert set(before.alleles) == {"cdna-d"} # links not yet live → class collapses to the focus + assert before.annotations["cdna-d"].vep is None + + after = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T2) + assert set(after.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} + assert after.annotations["cdna-d"].vep is not None diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py new file mode 100644 index 000000000..8822c928d --- /dev/null +++ b/tests/lib/test_allele_measurements.py @@ -0,0 +1,752 @@ +# ruff: noqa: E402 +"""Integration tests for the ClinGen-allele measurements list (``lib/allele_measurements.py``). + +These pin the model: measurements aggregate the **cross-layer equivalence class** (co-membership in the +mapping/RT link graph), not one exact allele, and *which* allele anchors it determines the direct/related +split (a CA anchors on the nt alleles → nt measurements are direct, the protein consequence and the sibling +nt encodings are related; a PA anchors on the protein → protein measurements are direct, the nt encodings +related). Also: the protein apex reaches a consequence whose assay isn't reverse-translated (with its +cardinality/provenance instrumentation), the two authorization gates (score-set READ hides the measurement; +calibration READ withholds only the inline classification) exercised through the real ``has_permission``, +superseded opt-in, and the default ordering (direct-first, strongest evidence, pathogenic-biased). +""" + +import logging + +import pytest + +pytest.importorskip("psycopg2") + +from datetime import date + +from starlette_context import context, request_cycle_context + +from mavedb.lib.allele_measurements import ( + AlleleMeasurement, + _ordering_key, + _resolve_protein_apex, + get_allele_measurements, +) +from mavedb.lib.mondo import get_generic_disease_term +from mavedb.lib.types.authentication import UserData +from mavedb.models.enums.measurement_relationship import MeasurementRelationship +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import EXTRA_USER, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, TEST_USER + + +def _user(session, username): + return session.query(User).filter(User.username == username).one() + + +def _user_data(session, username=TEST_USER["username"]): + """A ``UserData`` for a seeded user — the currency ``has_permission`` reads. ``None`` is anonymous.""" + return UserData(user=_user(session, username), active_roles=[]) + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None): + allele = Allele(vrs_digest=digest, level=level, clingen_allele_id=clingen_allele_id) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False): + session.add( + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + ) + ) + session.commit() + + +def _calibration( + session, + score_set, + *, + variants, + functional="abnormal", + oddspaths_ratio=None, + primary=True, + private=False, + investigator_provided=False, + research_use_only=False, +): + user = _user(session, TEST_USER["username"]) + calibration = ScoreCalibration( + score_set_id=score_set.id, + title="cal", + primary=primary, + private=private, + investigator_provided=investigator_provided, + research_use_only=research_use_only, + disease_term=get_generic_disease_term(session), + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label=functional, + functional_classification=functional, + oddspaths_ratio=oddspaths_ratio, + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +def _second_score_set( + session, base, urn, *, owner_username=TEST_USER["username"], private=True, superseded_score_set_id=None +): + """A sibling score set in the same experiment — for cross-score-set, private, and supersession tests. + ``superseded_score_set_id`` set on this (newer) row makes it the superseding version of that older + set (``older.superseding_score_set`` resolves back here).""" + owner = _user(session, owner_username) + scaffold = TEST_SEQ_SCORESET.copy() + scaffold.pop("target_genes", None) + score_set = ScoreSet( + **scaffold, + urn=urn, + experiment_id=base.experiment_id, + licence_id=base.licence_id, + superseded_score_set_id=superseded_score_set_id, + ) + score_set.private = private + score_set.created_by = owner + score_set.modified_by = owner + session.add(score_set) + session.commit() + session.refresh(score_set) + return score_set + + +def _seed_two_level_pair(session, score_set): + """One nt change (CA123) and its protein consequence (PA9), each measured in its own assay, with the + other level riding as a member — the minimal shape that exercises the direct/related asymmetry.""" + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna", hgvs_assay_level="NM_0:c.1A>T") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein", hgvs_assay_level="NP_0:p.Met1Leu") + _link(session, protein_record, prot, is_authoritative=True) + _link(session, protein_record, nt) + + return nt_measured, protein_measured + + +@pytest.mark.integration +def test_ca_entry_aggregates_direct_and_related(session, setup_lib_db_with_score_set): + """A CA page: the nt measurement is direct (assayed at this change); the protein measurement of its + consequence is related, labeled ``protein_consequence``, each carrying its own measured level.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert (by_urn[nt_measured.urn].relationship, by_urn[nt_measured.urn].assay_level) == ("direct", "cdna") + assert (by_urn[protein_measured.urn].relationship, by_urn[protein_measured.urn].assay_level) == ( + "protein_consequence", + "protein", + ) + # Assay HGVS pair rides through (mapped reference + submitted target). + assert by_urn[nt_measured.urn].assay_level_hgvs == "NM_0:c.1A>T" + assert by_urn[nt_measured.urn].submitted_hgvs == "c.1A>T" + assert by_urn[protein_measured.urn].submitted_hgvs == "p.Met1Leu" + + +@pytest.mark.integration +def test_pa_entry_swaps_direct_and_related(session, setup_lib_db_with_score_set): + """The same data anchored on the PA: the protein measurement is now direct and the nt measurement is + the related ``nucleotide_encoding`` — the asymmetry is purely which allele anchors the co-membership.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "direct" + assert by_urn[nt_measured.urn].relationship == "nucleotide_encoding" + + +@pytest.mark.integration +def test_sibling_nt_shown_on_ca_page(session, setup_lib_db_with_score_set): + """Reverse translation links every record to its full synonymous nt set, so a sibling nt change's record + links the anchor allele directly and the sibling shows on the CA page as ``nucleotide_encoding`` (after + the direct measurement). Both nt changes also show on the shared protein's PA page as encodings.""" + score_set = setup_lib_db_with_score_set + nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") + nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # Real RT linking: each record links its authoritative nt allele, the sibling nt allele (an RT + # candidate), and the shared protein consequence. + a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + ra = _record(session, a, assay_level="cdna") + _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, nt2) + _link(session, ra, prot) + + c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + rc = _record(session, c, assay_level="cdna") + _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, nt1) + _link(session, rc, prot) + + ca_page = get_allele_measurements(session, "CA111", user_data=_user_data(session)) + assert {m.variant_urn: m.relationship for m in ca_page} == {a.urn: "direct", c.urn: "nucleotide_encoding"} + # Direct precedes the sibling encoding. + assert [m.variant_urn for m in ca_page] == [a.urn, c.urn] + + pa_page = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + assert {m.variant_urn for m in pa_page} == {a.urn, c.urn} + assert all(m.relationship == "nucleotide_encoding" for m in pa_page) + + +@pytest.mark.integration +def test_resolve_protein_apex_single(session, setup_lib_db_with_score_set): + """The resolver returns the one protein consequence co-membered with the anchor nt allele, its PAID, + and no unregistered rows.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) + + apex = _resolve_protein_apex(session, [nt.id], as_of=None, score_set_ids={score_set.id}) + assert apex.allele_ids == [prot.id] + assert apex.caids == ["PA9"] + assert apex.unregistered == 0 + + +@pytest.mark.integration +def test_resolve_protein_apex_unregistered(session, setup_lib_db_with_score_set): + """An apex protein allele with no CAID yet counts toward ``unregistered`` and contributes no PAID.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id=None) + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) + + apex = _resolve_protein_apex(session, [nt.id], as_of=None, score_set_ids={score_set.id}) + assert apex.allele_ids == [prot.id] + assert apex.caids == [] + assert apex.unregistered == 1 + + +@pytest.mark.integration +def test_widening_ambiguous_apex_is_loud(session, setup_lib_db_with_score_set, caplog): + """An anchor whose nt change is co-membered with two *different* protein apexes (a divergent walk) is + surfaced loudly: a warning naming the conflicting PAIDs, and ``apex_paid_count == 2`` in the request + log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot1 = _allele(session, "prot-1", level="protein", clingen_allele_id="PA1") + prot2 = _allele(session, "prot-2", level="protein", clingen_allele_id="PA2") + + # nt is the measured allele of r1 (→ PA1) and a derived member of r2 (→ PA2): the same nt change + # resolving to two protein consequences. + v1 = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r1 = _record(session, v1, assay_level="cdna") + _link(session, r1, nt, is_authoritative=True) + _link(session, r1, prot1) + + v2 = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + r2 = _record(session, v2, assay_level="protein") + _link(session, r2, prot2, is_authoritative=True) + _link(session, r2, nt) + + with request_cycle_context({}): + with caplog.at_level(logging.WARNING, logger="mavedb.lib.allele_measurements"): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["apex_paid_count"] == 2 + + assert any("PA1" in rec.message and "PA2" in rec.message for rec in caplog.records) + + +@pytest.mark.integration +def test_widening_provenance_apex_only_unresolved(session, setup_lib_db_with_score_set): + """A protein measurement reached only through the apex, whose own record carries no derived link, is + counted apex-only-unresolved; giving that record a derived link of its own drops the count.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + # Both surface via the apex; the protein measurement is the apex-only-unresolved one. + assert {m.variant_urn for m in result} == {nt_measured.urn, protein_measured.urn} + assert context["measurements_protein_consequence"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 + + # Once the protein record gains a derived (non-authoritative) link of its own, it is resolved. + _link(session, protein_record, nt) + with request_cycle_context({}): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["measurements_apex_only_unresolved"] == 0 + + +@pytest.mark.integration +def test_ca_page_reaches_apex_only_protein_consequence(session, setup_lib_db_with_score_set): + """The variant page (no siblings) reaches a protein measurement of the anchor's consequence even when + that measurement's record has no nt link of its own (RT never ran on its score set) — through the + shared apex — and publishes the apex provenance to the request log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + by_urn = {m.variant_urn: m for m in result} + # Reached through the apex despite no nt link on the protein measurement's record. + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "protein_consequence" + # The page path now widens, so it emits the apex instrumentation. + assert context["apex_paid_count"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 + + +@pytest.mark.integration +def test_private_score_set_measurement_excluded(session, setup_lib_db_with_score_set): + """Score-set READ gates inclusion: a measurement in a score set the caller cannot read never leaks, + even though it links the same (public, content-addressed) allele.""" + score_set = setup_lib_db_with_score_set # private, owned by TEST_USER + other = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", owner_username=EXTRA_USER["username"]) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + + visible = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, visible, assay_level="cdna"), nt, is_authoritative=True) + hidden = _variant(session, other, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, hidden, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert {m.variant_urn for m in result} == {visible.urn} + + +@pytest.mark.integration +@pytest.mark.parametrize("ca_also_measured_publicly", [False, True]) +def test_protein_consequence_reached_only_through_unreadable_score_set_is_withheld( + session, setup_lib_db_with_score_set, ca_also_measured_publicly +): + """A protein consequence that only an unreadable score set's mapping links to the CA must not pull in + public measurements of that protein change: they would disclose the private record.""" + private = setup_lib_db_with_score_set # private, owned by TEST_USER + public = _second_score_set(session, private, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # The private score set's reverse translation is the only record linking CA123 to PA9. + private_measured = _variant(session, private, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + private_record = _record(session, private_measured, assay_level="cdna") + _link(session, private_record, nt, is_authoritative=True) + _link(session, private_record, prot) + + # A public protein assay of PA9, with no reverse translation of its own. + public_protein = _variant(session, public, 1, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + _link(session, _record(session, public_protein, assay_level="protein"), prot, is_authoritative=True) + + expected: set[str] = set() + if ca_also_measured_publicly: + # CA123 is itself readable through the public set, but that record doesn't reach PA9. + public_nt = _variant(session, public, 2, data={"score_data": {"score": 0.5}}, hgvs_nt="c.1A>T") + _link(session, _record(session, public_nt, assay_level="cdna"), nt, is_authoritative=True) + expected = {public_nt.urn} + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=None) + + assert {m.variant_urn for m in result} == expected + + +@pytest.mark.integration +def test_calibration_gate_withholds_classification(session, setup_lib_db_with_score_set): + """Calibration READ gates only the inline classification: on a public score set with a private + calibration, an anonymous caller still sees the measurement but its classification is withheld, while + the owner sees both.""" + score_set = setup_lib_db_with_score_set + public = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, public, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, public, variants=[variant], functional="abnormal", private=True) + + anonymous = get_allele_measurements(session, "CA123", user_data=None) + assert len(anonymous) == 1 + assert anonymous[0].classification is None + + owner = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert owner[0].classification is not None + assert owner[0].classification.functional_classification.value == "abnormal" + + +@pytest.mark.integration +def test_preferred_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): + """With no primary calibration, the preference cascade dominates evidence strength: an + investigator-provided calibration wins over a stronger non-RUO one and a stronger-still RUO one.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) # plain, non-RUO + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0, investigator_provided=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + # Investigator-provided (weakest) is chosen over the stronger non-RUO and RUO calibrations. + assert result[0].classification.oddspaths_ratio == 2.0 + + +@pytest.mark.integration +def test_preferred_classification_strongest_within_tier(session, setup_lib_db_with_score_set): + """Within one cascade tier (here two plain non-RUO calibrations), the strongest evidence wins.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].classification.oddspaths_ratio == 100.0 + + +@pytest.mark.integration +def test_research_use_only_calibrations_are_excluded(session, setup_lib_db_with_score_set): + """A research-use-only calibration is excluded from the cascade (even if it is the strongest + evidence).""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert result[0].classification is None + + # Opted in, the RUO pick stands in, by the same cascade (strongest evidence within the tier). + opted = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + assert opted[0].classification.oddspaths_ratio == 1000.0 + assert opted[0].classification_is_research_use_only + + +@pytest.mark.integration +def test_research_use_only_classification_absent_when_a_preferred_call_exists(session, setup_lib_db_with_score_set): + """The RUO fallback never displaces a non-RUO call, even when the RUO call + is stronger.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + + assert result[0].classification.oddspaths_ratio == 2.0 + assert not result[0].classification_is_research_use_only + + +@pytest.mark.integration +def test_research_use_only_classification_absent_when_a_clinical_calibration_leaves_the_variant_unclassified( + session, setup_lib_db_with_score_set +): + """A non-RUO calibration that doesn't classify the variant still blocks the RUO fallback: the clinical + calibration's silence is its answer.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + other = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[other], primary=True, oddspaths_ratio=2.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + + assert result[0].classification is None + + +@pytest.mark.integration +def test_research_use_only_classification_falls_back_past_a_clinical_calibration_without_classifications( + session, setup_lib_db_with_score_set +): + """A non-RUO calibration with no functional classifications can't classify anything, so it doesn't block the + RUO fallback.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + empty = _calibration(session, score_set, variants=[], primary=True) + for classification in empty.functional_classifications: + session.delete(classification) + session.commit() + session.refresh(score_set) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + + assert result[0].classification.oddspaths_ratio == 1000.0 + assert result[0].classification_is_research_use_only + + +@pytest.mark.integration +def test_research_use_only_classification_falls_back_past_an_unreadable_clinical_calibration( + session, setup_lib_db_with_score_set +): + """A non-RUO calibration the caller can't read doesn't count, so the RUO fallback still shows.""" + public = _second_score_set(session, setup_lib_db_with_score_set, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, public, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, public, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, public, variants=[variant], primary=False, oddspaths_ratio=2.0, private=True) + + result = get_allele_measurements(session, "CA123", user_data=None, include_research_use_only=True) + + assert result[0].classification.oddspaths_ratio == 1000.0 + assert result[0].classification_is_research_use_only + + +@pytest.mark.integration +def test_research_use_only_classification_does_not_rank(session, setup_lib_db_with_score_set): + """A measurement carrying only an RUO call ranks as unclassified: below one with a non-RUO call, however + strong the RUO evidence.""" + score_set = setup_lib_db_with_score_set + ruo_score_set = _second_score_set(session, score_set, "urn:mavedb:00000001-a-2", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + ruo_only = _variant(session, ruo_score_set, 1, data={"score_data": {"score": 1.0}}) + classified = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, ruo_only, assay_level="cdna"), nt, is_authoritative=True) + _link(session, _record(session, classified, assay_level="cdna"), nt, is_authoritative=True) + + _calibration( + session, ruo_score_set, variants=[ruo_only], primary=False, oddspaths_ratio=1000.0, research_use_only=True + ) + _calibration(session, score_set, variants=[classified], primary=False, oddspaths_ratio=2.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_research_use_only=True) + + assert [m.variant_urn for m in result] == [classified.urn, ruo_only.urn] + assert result[1].classification_is_research_use_only + + +@pytest.mark.integration +def test_superseded_excluded_by_default_included_opt_in(session, setup_lib_db_with_score_set): + """Superseded measurements are current-tail-only by default; ``include_superseded`` opts in and they + self-describe (``is_current`` false + the superseding score set's URN).""" + score_set = setup_lib_db_with_score_set + _second_score_set(session, score_set, "urn:mavedb:00000001-a-2", superseded_score_set_id=score_set.id) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + assert get_allele_measurements(session, "CA123", user_data=_user_data(session)) == [] + + opted = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_superseded=True) + assert len(opted) == 1 + assert opted[0].is_current is False + assert opted[0].superseded_by_score_set == "urn:mavedb:00000001-a-2" + + +@pytest.mark.integration +def test_unreadable_superseding_reads_as_current(session, setup_lib_db_with_score_set): + """A superseding version the caller cannot read is not leaked — the measurement reads as current and + is included by default.""" + score_set = setup_lib_db_with_score_set # owned by TEST_USER + _second_score_set( + session, + score_set, + "urn:mavedb:00000001-a-2", + owner_username=EXTRA_USER["username"], + superseded_score_set_id=score_set.id, + ) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert len(result) == 1 + assert result[0].is_current is True + assert result[0].superseded_by_score_set is None + + +@pytest.mark.integration +def test_ordering_direct_first_then_strongest_pathogenic(session, setup_lib_db_with_score_set): + """Default order: direct before related; within a group strongest evidence first, pathogenic before + benign at equal magnitude; the unclassified related measurement sorts last.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + strong_path = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_path], functional="abnormal", oddspaths_ratio=100.0) + + strong_benign = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_benign, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_benign], functional="normal", oddspaths_ratio=0.01) + + weak_path = _variant(session, score_set, 3, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, weak_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[weak_path], functional="abnormal", oddspaths_ratio=2.0) + + related = _variant(session, score_set, 4, data={"score_data": {"score": 1.0}}) + related_record = _record(session, related, assay_level="protein") + _link(session, related_record, prot, is_authoritative=True) + _link(session, related_record, nt) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert [m.variant_urn for m in result] == [strong_path.urn, strong_benign.urn, weak_path.urn, related.urn] + + +@pytest.mark.integration +def test_unknown_clingen_id_returns_empty(session, setup_lib_db_with_score_set): + assert get_allele_measurements(session, "CA000", user_data=_user_data(session)) == [] + + +def _oddspaths_classification(ratio): + """A transient functional classification carrying only an oddspaths ratio — enough for + ``classification_evidence_strength`` (which reads ``acmg_classification`` [None here] then the ratio) + to rank it, with no DB round-trip.""" + return ScoreCalibrationFunctionalClassification(oddspaths_ratio=ratio) + + +def _measurement(urn, *, relationship=MeasurementRelationship.direct, classification=None, is_current=True): + """A transit ``AlleleMeasurement`` with just the fields ``_ordering_key`` reads; the rest are filler.""" + return AlleleMeasurement( + variant_urn=urn, + score=None, + assay_level=None, + relationship=relationship, + assay_level_hgvs=None, + submitted_hgvs=None, + score_set_urn="", + score_set_title="", + classification=classification, + classification_is_research_use_only=False, + is_current=is_current, + superseded_by_score_set=None, + ) + + +def test_ordering_key_ranks_every_dimension(): + """Pin the full precedence chain of ``_ordering_key`` as a pure unit (no DB, no calibration fixtures): + current → direct → classified → strongest magnitude → pathogenic direction → most assay facts → newest + published → urn. + Each adjacent pair below differs by exactly the next-lower key, so sorting a shuffled copy must + reproduce this exact order.""" + strong_path = _oddspaths_classification(100.0) # magnitude |ln 100|, pathogenic + strong_benign = _oddspaths_classification(0.01) # equal magnitude, benign + weak_path = _oddspaths_classification(2.0) # smaller magnitude, pathogenic + + # (measurement, published_date, assay_fact_count), already in the expected ranked order. + ranked = [ + (_measurement("u#1", classification=strong_path), date(2024, 1, 1), 0), # strongest evidence, pathogenic + (_measurement("u#2", classification=strong_benign), date(2024, 1, 1), 0), # equal magnitude, benign loses + (_measurement("u#3", classification=weak_path), date(2024, 1, 1), 0), # weaker magnitude + (_measurement("u#4", classification=None), date(2024, 1, 1), 3), # unclassified; most assay facts leads + (_measurement("u#5a", classification=None), date(2024, 6, 1), 0), # fewer facts; newer published leads + (_measurement("u#5b", classification=None), date(2024, 1, 1), 0), # unclassified, older published + (_measurement("u#5c", classification=None), date(2024, 1, 1), 0), # same date → urn tiebreak + (_measurement("u#6", relationship=MeasurementRelationship.protein_consequence), date(2024, 6, 1), 3), # related + (_measurement("u#7", is_current=False), date(2024, 6, 1), 3), # superseded sorts last + ] + + shuffled = [ranked[i] for i in (5, 0, 8, 7, 2, 4, 1, 6, 3)] + ordered = sorted(shuffled, key=lambda entry: _ordering_key(*entry)) + + assert [m.variant_urn for m, _, _ in ordered] == [m.variant_urn for m, _, _ in ranked] + + +def test_ordering_key_missing_published_date_sorts_after_dated(): + """A null published date is treated as oldest (``0`` ordinal), so a dated measurement leads an + otherwise-identical undated one.""" + dated = (_measurement("u#dated"), date(2024, 1, 1), 0) + undated = (_measurement("u#undated"), None, 0) + + ordered = sorted([undated, dated], key=lambda entry: _ordering_key(*entry)) + + assert [m.variant_urn for m, _, _ in ordered] == ["u#dated", "u#undated"] diff --git a/tests/lib/test_alleles.py b/tests/lib/test_alleles.py new file mode 100644 index 000000000..ed8c277d5 --- /dev/null +++ b/tests/lib/test_alleles.py @@ -0,0 +1,198 @@ +# ruff: noqa: E402 +"""Integration tests for the record-scoped allele-link query backing Cat-VRS transit. + +``get_live_record_allele_links`` stays within each variant's own live ``MappingRecord`` — unlike +``get_allele_translations``, which takes the cross-record union an anchor allele can belong to. These +tests pin that scoping, the ValidTime live/retired filtering, and the ``as_of`` reconstruction. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# Deterministic windows far from the transaction clock, so an accidental func.now() stamp is visibly +# wrong rather than coincidentally equal (mirrors tests/db/test_mixins.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _record(session, variant, *, assay_level="genomic", valid_from=None): + record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level=assay_level, + mapping_api_version="test.0.0", + valid_from=valid_from, + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + valid_from=valid_from, + ) + session.add(link) + session.commit() + return link + + +def _digests(links): + return {link.allele.vrs_digest for link in links} + + +def _links(session, variant_id, **kwargs): + """One variant's live links through the batched loader; empty when it has no live record.""" + entry = get_live_record_allele_links(session, [variant_id], **kwargs).get(variant_id) + return entry.links if entry is not None else [] + + +@pytest.mark.integration +def test_returns_live_links_with_alleles(session, setup_lib_db_with_score_set): + """The variant's live record yields its authoritative + derived links, each allele eagerly loaded.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + record = _record(session, variant) + authoritative = _allele(session, "auth", level="genomic") + derived = _allele(session, "deriv", level="protein") + _link(session, record, authoritative, is_authoritative=True) + _link(session, record, derived, is_authoritative=False) + + links = _links(session, variant.id) + + assert _digests(links) == {"auth", "deriv"} + # Exactly one authoritative (the defining allele); the eager-loaded allele is reachable. + assert [link.allele.vrs_digest for link in links if link.is_authoritative] == ["auth"] + + +@pytest.mark.integration +def test_no_mapping_record_returns_empty(session, setup_lib_db_with_score_set): + assert get_live_record_allele_links(session, [1]) == {} + + +@pytest.mark.integration +def test_excludes_a_retired_link(session, setup_lib_db_with_score_set): + """A link retired within a still-live record is not part of the current set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + record = _record(session, variant) + live = _allele(session, "live") + stale = _allele(session, "stale") + _link(session, record, live, is_authoritative=True) + retired = _link(session, record, stale) + retired.retire(at=T1) + session.commit() + + assert _digests(_links(session, variant.id)) == {"live"} + + +@pytest.mark.integration +def test_excludes_a_superseded_record(session, setup_lib_db_with_score_set): + """A re-map retires the prior record (cascading to its links); only the new record's links surface.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + old_allele = _allele(session, "old") + new_allele = _allele(session, "new") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + + assert _digests(_links(session, variant.id)) == {"new"} + + +@pytest.mark.integration +def test_record_scoped_not_cross_record_union(session, setup_lib_db_with_score_set): + """A shared allele linked by two variants' records does NOT pull the other record's members in — + this is the deliberate difference from get_allele_translations' cross-record union.""" + score_set = setup_lib_db_with_score_set + shared = _allele(session, "shared") + only_a = _allele(session, "only_a") + only_b = _allele(session, "only_b") + + variant_a = _variant(session, score_set, 1) + record_a = _record(session, variant_a) + _link(session, record_a, shared, is_authoritative=True) + _link(session, record_a, only_a) + + variant_b = _variant(session, score_set, 2) + record_b = _record(session, variant_b) + _link(session, record_b, shared, is_authoritative=True) + _link(session, record_b, only_b) + + # Variant A's record sees the shared allele and its own member, never variant B's. + assert _digests(_links(session, variant_a.id)) == {"shared", "only_a"} + + +@pytest.mark.integration +def test_as_of_returns_the_historical_link_set(session, setup_lib_db_with_score_set): + """as_of reconstructs the record + links live at a past instant, not the current re-mapped set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + past_allele = _allele(session, "past") + current_allele = _allele(session, "current") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, past_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, current_allele, is_authoritative=True, valid_from=T1) + + # Inside the old window: the historical set. + assert _digests(_links(session, variant.id, as_of=T0)) == {"past"} + assert _digests(_links(session, variant.id, as_of=T1 - timedelta(days=1))) == {"past"} + # At/after the handoff and current: the new set. + assert _digests(_links(session, variant.id, as_of=T2)) == {"current"} + assert _digests(_links(session, variant.id)) == {"current"} + + +@pytest.mark.integration +def test_one_call_returns_each_variants_own_record_and_links(session, setup_lib_db_with_score_set): + """The batched form keys each variant to its own record and links, and omits a variant with none.""" + score_set = setup_lib_db_with_score_set + variant_a = _variant(session, score_set, 1) + variant_b = _variant(session, score_set, 2) + unmapped = _variant(session, score_set, 3) + record_a = _record(session, variant_a) + record_b = _record(session, variant_b) + _link(session, record_a, _allele(session, "a"), is_authoritative=True) + _link(session, record_b, _allele(session, "b"), is_authoritative=True) + + result = get_live_record_allele_links(session, [variant_a.id, variant_b.id, unmapped.id]) + + assert set(result) == {variant_a.id, variant_b.id} + assert result[variant_a.id].record.id == record_a.id + assert _digests(result[variant_a.id].links) == {"a"} + assert _digests(result[variant_b.id].links) == {"b"} diff --git a/tests/lib/test_annotation_status_manager.py b/tests/lib/test_annotation_status_manager.py index 52771b6bf..354ed85fc 100644 --- a/tests/lib/test_annotation_status_manager.py +++ b/tests/lib/test_annotation_status_manager.py @@ -5,1362 +5,278 @@ pytest.importorskip("psycopg2") from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.models.allele import Allele from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus -from mavedb.models.variant import Variant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason @pytest.fixture def annotation_status_manager(session, job_run): - """Fixture to provide an AnnotationStatusManager instance.""" return AnnotationStatusManager(session, job_run_id=job_run.id) @pytest.fixture -def existing_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.fixture -def existing_unversioned_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() +def allele(session): + allele = Allele(vrs_digest="asm-test-allele-digest", level="genomic") + session.add(allele) session.commit() + session.refresh(allele) + return allele - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.mark.unit -class TestAnnotationStatusManagerCreateAnnotationUnit: - """Unit tests for AnnotationStatusManager.add_annotation method.""" - - @pytest.mark.parametrize( - "annotation_type", - AnnotationType._member_map_.values(), - ) - @pytest.mark.parametrize( - "status", - AnnotationStatus._member_map_.values(), - ) - def test_add_annotation_creates_entry_with_annotation_type_version_status( - self, session, annotation_status_manager, annotation_type, status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry with correct type and version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - annotation_data={}, - current=True, - status=status, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - ) - - assert annotation is not None - assert annotation.annotation_type == annotation_type - assert annotation.status == status - assert annotation.version == "v1.0" - - def test_add_annotation_stores_job_run_id( - self, session, annotation_status_manager, job_run, setup_lib_db_with_variant - ): - """Test that every annotation is created with the job_run_id from the manager.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - annotation_data={}, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.job_run_id == job_run.id - - def test_add_annotation_persists_annotation_data( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding an annotation persists the provided annotation data.""" - annotation_data = { - "annotation_metadata": {"some_key": "some_value"}, - "error_message": None, - } - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - failure_category=None, - annotation_data=annotation_data, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.failure_category is None - for key, value in annotation_data.items(): - assert getattr(annotation, key) == value - - def test_add_annotation_creates_entry_and_marks_previous_not_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry and marks previous ones as not current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type, version) - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - def test_add_annotation_with_different_version_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different version keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type) but different version - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_with_different_type_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different type keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same variant but different type - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_without_version(self, session, annotation_status_manager, setup_lib_db_with_variant): - """Test that adding an annotation without specifying version works correctly.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - version=None, - annotation_data={}, - status=AnnotationStatus.SKIPPED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.version is None - assert annotation.current is True - - def test_add_annotation_multiple_without_version_marks_previous_not_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding multiple annotations without version marks previous ones as not current.""" - - # Add second annotation without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is False - - def test_add_annotation_different_type_without_version_keeps_previous_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation of different type without version keeps previous current.""" - - # Add second annotation of different type without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is True - - def test_add_annotation_multiple_variants_independent_current_flags( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants maintains independent current flags.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation1 is not None - assert annotation1.id is not None - assert annotation1.current is True - - assert annotation2 is not None - assert annotation2.id is not None - assert annotation2.current is True - - -class TestAnnotationStatusManagerGetCurrentAnnotationUnit: - """Unit tests for AnnotationStatusManager.get_current_annotation method.""" - - def test_get_current_annotation_returns_none_when_no_entry( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when no entry exists.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_returns_correct_entry( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation.id == existing_annotation_status.id - assert annotation.current is True - - def test_get_current_annotation_returns_none_for_non_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when the entry is not current.""" - # Mark existing annotation as not current - existing_annotation_status.current = False - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_with_different_version_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different version returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert annotation is None - - def test_get_current_annotation_with_different_type_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different type returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_without_version_returns_correct_entry( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation without version returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - ) - assert annotation.id == existing_unversioned_annotation_status.id - assert annotation.current is True - - -class TestAnnotationStatusManagerIntegration: - """Integration tests for AnnotationStatusManager methods.""" - - def test_add_and_get_current_annotation_work_together( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding and getting current annotation work together correctly.""" - # Add annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_multiple_and_get_current_returns_latest( - self, session, annotation_status_manager, version, setup_lib_db_with_variant - ): - """Test that adding multiple annotations and getting current returns the latest one.""" - # Add first annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Add second annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.version == version - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_annotations_for_different_variants_and_get_current_independent( - self, session, annotation_status_manager, version, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants and getting current works independently.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation for variant 1 - retrieved_annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation1 is not None - assert retrieved_annotation1.current is True - assert retrieved_annotation1.status == AnnotationStatus.SUCCESS - assert retrieved_annotation1.version == version - - # Get current annotation for variant 2 - retrieved_annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation2 is not None - assert retrieved_annotation2.current is True - assert retrieved_annotation2.status == AnnotationStatus.FAILED - assert retrieved_annotation2.version == version - - -@pytest.mark.unit -class TestAnnotationStatusManagerReplaceAllVersionsUnit: - """Unit tests for the replace_all_versions parameter of AnnotationStatusManager.add_annotation.""" - - def test_replace_all_versions_false_keeps_different_version_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Default behavior: a new annotation only retires the same version, not others.""" - # existing_annotation_status is version "v1", current=True - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_annotation is not None - assert new_annotation.current is True - - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_replace_all_versions_true_retires_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True retires all current records for (variant, type) regardless of version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - # Both v1 and v2 are current at this point (replace_all_versions=False) - v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert v1 is not None and v1.current is True - assert v2 is not None and v2.current is True - - # Now add v3 with replace_all_versions=True — should retire both v1 and v2 - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(v1) - session.refresh(v2) - assert v1.current is False - assert v2.current is False - - v3 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", - ) - assert v3 is not None and v3.current is True - def test_replace_all_versions_true_only_affects_matching_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True only retires records for the same annotation_type.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( +class TestRecordEvent: + def test_records_variant_subject_event(self, session, annotation_status_manager, setup_lib_db_with_variant): + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) annotation_status_manager.flush() session.commit() - vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", + event = annotation_status_manager.get_current_annotation( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id ) + assert event is not None + assert event.disposition == Disposition.PRESENT + assert event.allele_id is None + # No current flag exists on the event log. + assert not hasattr(event, "current") - # replace VRS_MAPPING only - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + def test_records_allele_subject_event_with_metadata(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", + metadata={"gnomad_variant_id": "1-55051215-G-A"}, ) annotation_status_manager.flush() session.commit() - session.refresh(vrs) - session.refresh(clinvar) - assert vrs.current is False - assert clinvar.current is True - - new_vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_vrs is not None and new_vrs.current is True - - def test_replace_all_versions_true_only_affects_matching_variant( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """replace_all_versions=True only retires records for the same variant_id.""" - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - ann2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - # replace variant1 only - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - annotation_status_manager.flush() - session.commit() + assert event.event_metadata == {"gnomad_variant_id": "1-55051215-G-A"} + assert event.source_version == "4.1.0" + assert event.variant_id is None - session.refresh(ann1) - session.refresh(ann2) - assert ann1.current is False - assert ann2.current is True # untouched - new_ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_ann1 is not None and new_ann1.current is True - - def test_replace_all_versions_true_same_version_also_retired( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """replace_all_versions=True retires a same-version record just as replace_all_versions=False would.""" - # existing_annotation_status is version "v1" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=True, - ) +class TestAppendOnlyLatestWins: + def test_latest_event_by_id_is_current(self, session, annotation_status_manager, allele): + for reason in (EventReason.CREATED, EventReason.RECONFIRMED, EventReason.SKIPPED): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=reason, + source_version="4.1.0", + ) annotation_status_manager.flush() session.commit() - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + # All three events persist (append-only — no retire of prior rows). + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert new_annotation is not None - assert new_annotation.current is True - assert new_annotation.status == AnnotationStatus.FAILED - - -@pytest.mark.unit -class TestAnnotationStatusManagerBatchingUnit: - """Unit tests for batching and flush behavior.""" - - def test_flush_noop_when_empty(self, annotation_status_manager): - """flush() with no pending annotations does nothing and does not error.""" - annotation_status_manager.flush() # should not raise - - def test_auto_flush_at_batch_size(self, session, setup_lib_db_with_score_set): - """Annotations are auto-flushed to the DB when batch_size is reached.""" - variants = [ - Variant(score_set_id=1, hgvs_nt=f"NM_000000.1:c.{i}A>G", hgvs_pro=f"NP_000000.1:p.Met{i}Val", data={}) - for i in range(3) + assert [e.reason for e in history] == [ # newest first + EventReason.SKIPPED, + EventReason.RECONFIRMED, + EventReason.CREATED, ] - session.add_all(variants) - session.commit() - for v in variants: - session.refresh(v) - - manager = AnnotationStatusManager(session, batch_size=2) - - # Add first — stays pending (below threshold) - manager.add_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - - # Add second — triggers auto-flush (reaches batch_size=2) - manager.add_annotation( - variant_id=variants[1].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 0 # flushed - - # Verify the auto-flushed rows are visible in the DB - ann = manager.get_current_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann is not None and ann.current is True - - # Add a third — stays pending (below threshold again) - manager.add_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - # Explicit flush persists the remainder - manager.flush() - assert len(manager._pending) == 0 - - ann3 = manager.get_current_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann3 is not None and ann3.current is True - - def test_get_current_annotation_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_current_annotation() flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush — get_current_annotation should auto-flush - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is not None - assert annotation.current is True + # Current = newest by id. + current = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert current.reason == EventReason.SKIPPED + + def test_source_version_scopes_current(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.0.0", + ) + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", + ) + annotation_status_manager.flush() + session.commit() + + v40 = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id, source_version="4.0.0" + ) + assert v40.disposition == Disposition.PRESENT + + +class TestSubjectValidation: + def test_variant_subject_type_with_allele_id_raises(self, annotation_status_manager, allele): + with pytest.raises(ValueError, match="variant-subject"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_allele_subject_type_with_variant_id_raises(self, annotation_status_manager, setup_lib_db_with_variant): + with pytest.raises(ValueError, match="allele-subject"): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_neither_subject_raises(self, annotation_status_manager): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_both_subjects_raises(self, annotation_status_manager, setup_lib_db_with_variant, allele): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + +class TestQueryReads: + def test_get_current_annotation_returns_none_when_no_event(self, annotation_status_manager, allele): + # No event recorded for this subject yet — current status is None, not an error. + assert ( + annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + is None + ) + + def test_get_event_history_empty_for_no_records(self, annotation_status_manager, allele): + assert ( + annotation_status_manager.get_event_history(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id) + == [] + ) + + def test_flush_is_noop_when_empty(self, annotation_status_manager): + # Nothing pending — flush must not raise or emit a write. assert len(annotation_status_manager._pending) == 0 - - def test_flush_clears_internal_buffers(self, session, annotation_status_manager, setup_lib_db_with_variant): - """flush() clears both _pending and _retirement_filters.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(annotation_status_manager._pending) == 1 - assert len(annotation_status_manager._retirement_filters) == 1 - annotation_status_manager.flush() assert len(annotation_status_manager._pending) == 0 - assert len(annotation_status_manager._retirement_filters) == 0 - - def test_batch_retirement_groups_by_annotation_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Multiple annotation types in one batch are retired independently.""" - # Create initial annotations for two types - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - vrs_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - ) - - # Now add replacements for both types in one batch - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(vrs_v1) - session.refresh(clinvar_v1) - assert vrs_v1.current is False - assert clinvar_v1.current is False - - vrs_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - clinvar_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - ) - assert vrs_v2 is not None and vrs_v2.current is True - assert clinvar_v2 is not None and clinvar_v2.current is True - - -@pytest.mark.unit -class TestAnnotationStatusManagerAuditHelpersUnit: - """Unit tests for audit query helpers: get_annotation_history and get_all_current_annotations.""" - - def test_get_annotation_history_returns_all_rows_newest_first( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history returns both current and retired rows, newest first.""" - # Create two annotations for the same (variant, type, version) — first gets retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert len(history) == 2 - # Newest first - assert history[0].status == AnnotationStatus.FAILED - assert history[0].current is True - assert history[1].status == AnnotationStatus.SUCCESS - assert history[1].current is False - - def test_get_annotation_history_filters_by_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history with version only returns matching rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - history_jan = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - ) - assert len(history_jan) == 1 - assert history_jan[0].version == "2025-01" - - def test_get_annotation_history_without_version_returns_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history without version returns rows across all versions.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, + def test_get_current_annotation_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, ) - assert len(history) == 2 - - def test_get_annotation_history_empty_for_no_records(self, annotation_status_manager, setup_lib_db_with_variant): - """get_annotation_history returns empty list when no records exist.""" - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, + # Buffered, never explicitly flushed — the read must flush first and still see it. + assert len(annotation_status_manager._pending) == 1 + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert history == [] + assert event is not None + assert len(annotation_status_manager._pending) == 0 - def test_get_annotation_history_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + def test_get_event_history_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, ) - # No explicit flush - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, + assert len(annotation_status_manager._pending) == 1 + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) assert len(history) == 1 assert len(annotation_status_manager._pending) == 0 - def test_get_all_current_annotations_returns_all_types( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns current annotations across all types.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + def test_failed_disposition_round_trips_through_manager(self, session, annotation_status_manager, allele): + # The failure path (the case the audit log most needs to capture) survives a write/read cycle + # with its disposition, reason, and error metadata intact. + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "upstream timeout"}, ) annotation_status_manager.flush() session.commit() - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 3 - types = {a.annotation_type for a in all_current} - assert types == { - AnnotationType.VRS_MAPPING, - AnnotationType.CLINVAR_CONTROL, - AnnotationType.CLINGEN_ALLELE_ID, - } - - def test_get_all_current_annotations_excludes_retired( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations does not include retired rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - annotation_status_manager.flush() + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.event_metadata == {"error_message": "upstream timeout"} - # Replace it — v1 becomes retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() + def test_current_is_isolated_per_subject(self, session, annotation_status_manager, allele): + # Recording for one allele must not bleed into another's current status. + other = Allele(vrs_digest="asm-test-allele-digest-2", level="genomic") + session.add(other) session.commit() + session.refresh(other) - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 1 - assert all_current[0].version == "v2" - - def test_get_all_current_annotations_empty_for_no_records( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns empty list when no records exist.""" - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert result == [] - - def test_get_all_current_annotations_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(result) == 1 - assert len(annotation_status_manager._pending) == 0 - - def test_get_all_current_annotations_ordered_by_type_then_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns results ordered by annotation_type, version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=other.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", ) annotation_status_manager.flush() session.commit() - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, + a = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert len(all_current) == 3 - # clinvar_control < vrs_mapping alphabetically - assert all_current[0].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[0].version == "2025-01" - assert all_current[1].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[1].version == "2025-02" - assert all_current[2].annotation_type == AnnotationType.VRS_MAPPING + b = annotation_status_manager.get_current_annotation(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=other.id) + assert a.disposition == Disposition.PRESENT + assert b.disposition == Disposition.ABSENT + assert a.allele_id != b.allele_id -@pytest.mark.unit -class TestVariantAnnotationStatusReprUnit: - """Unit tests for the VariantAnnotationStatus __repr__ method.""" - - def test_repr_includes_key_fields(self, session, annotation_status_manager, setup_lib_db_with_variant): - """__repr__ includes id, variant_id, type, version, status, current, and created_at.""" - annotation_status_manager.add_annotation( +class TestBatching: + def test_auto_flush_at_batch_size(self, session, job_run, setup_lib_db_with_variant, annotation_status_manager): + annotation_status_manager.batch_size = 2 + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( + # Not yet flushed (1 < batch_size). + assert len(annotation_status_manager._pending) == 1 + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) - repr_str = repr(annotation) - - assert "VariantAnnotationStatus" in repr_str - assert f"id={annotation.id}" in repr_str - assert f"variant_id={setup_lib_db_with_variant.id}" in repr_str - assert "type='vrs_mapping'" in repr_str - assert "version='v1'" in repr_str - assert "status='success'" in repr_str - assert "current=True" in repr_str - assert "created_at=" in repr_str + # Auto-flushed at batch_size=2. + assert len(annotation_status_manager._pending) == 0 + session.commit() diff --git a/tests/lib/test_authentication.py b/tests/lib/test_authentication.py index 80ee503cb..248e34642 100644 --- a/tests/lib/test_authentication.py +++ b/tests/lib/test_authentication.py @@ -1,5 +1,6 @@ # ruff: noqa: E402 +from datetime import datetime, timedelta from unittest.mock import patch import pytest @@ -9,7 +10,11 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") -from mavedb.lib.authentication import get_current_user, get_current_user_data_from_api_key +from mavedb.lib.authentication import ( + LAST_LOGIN_REFRESH_INTERVAL, + get_current_user, + get_current_user_data_from_api_key, +) from mavedb.models.enums.user_role import UserRole from mavedb.models.user import User from tests.helpers.constants import ADMIN_USER, ADMIN_USER_DECODED_JWT, TEST_USER, TEST_USER_DECODED_JWT @@ -17,61 +22,53 @@ from tests.helpers.util.user import mark_user_inactive -@pytest.mark.asyncio -async def test_get_current_user_data_from_key_valid_token(session, setup_lib_db): +def test_get_current_user_data_from_key_valid_token(session, setup_lib_db): access_key = create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, access_key) + user_data = get_current_user_data_from_api_key(session, access_key) assert user_data.user.username == TEST_USER["username"] -@pytest.mark.asyncio -async def test_get_current_user_data_from_key_invalid_token(session, setup_lib_db): +def test_get_current_user_data_from_key_invalid_token(session, setup_lib_db): access_key = create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, f"invalid_{access_key}") + user_data = get_current_user_data_from_api_key(session, f"invalid_{access_key}") assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_data_from_key_nonetype_token(session, setup_lib_db): +def test_get_current_user_data_from_key_nonetype_token(session, setup_lib_db): create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, None) + user_data = get_current_user_data_from_api_key(session, None) assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_via_api_key(session, setup_lib_db): +def test_get_current_user_via_api_key(session, setup_lib_db): access_key = create_api_key_for_user(session, TEST_USER["username"]) - user_data = await get_current_user_data_from_api_key(session, access_key) + user_data = get_current_user_data_from_api_key(session, access_key) - user_data = await get_current_user(user_data, None, session, None) + user_data = get_current_user(user_data, None, session, None) assert user_data.user.username == TEST_USER["username"] -@pytest.mark.asyncio -async def test_get_current_user_via_token_payload(session, setup_lib_db): - user_data = await get_current_user(None, TEST_USER_DECODED_JWT, session, None) +def test_get_current_user_via_token_payload(session, setup_lib_db): + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) assert user_data.user.username == TEST_USER["username"] -@pytest.mark.asyncio -async def test_get_current_user_no_api_no_jwt(session, setup_lib_db): - user_data = await get_current_user(None, None, session, None) +def test_get_current_user_no_api_no_jwt(session, setup_lib_db): + user_data = get_current_user(None, None, session, None) assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_no_username(session, setup_lib_db): +def test_get_current_user_no_username(session, setup_lib_db): # Remove the username key from the JWT jwt_without_sub = TEST_USER_DECODED_JWT.copy() jwt_without_sub.pop("sub") - user_data = await get_current_user(None, jwt_without_sub, session, None) + user_data = get_current_user(None, jwt_without_sub, session, None) assert user_data is None -@pytest.mark.asyncio @pytest.mark.parametrize("with_email", [True, False]) -async def test_get_current_user_nonexistent_user(session, setup_lib_db, with_email): +def test_get_current_user_nonexistent_user(session, setup_lib_db, with_email): new_user_jwt = { "sub": "5555-5555-5555-5555", "given_name": "Temporary", @@ -81,7 +78,7 @@ async def test_get_current_user_nonexistent_user(session, setup_lib_db, with_ema email = "tempemail@test.com" if with_email else None with patch("mavedb.lib.authentication.fetch_orcid_user_email") as orc_email: orc_email.return_value = email - user_data = await get_current_user(None, new_user_jwt, session, None) + user_data = get_current_user(None, new_user_jwt, session, None) orc_email.assert_called_once() assert user_data.user.username == new_user_jwt["sub"] @@ -93,39 +90,34 @@ async def test_get_current_user_nonexistent_user(session, setup_lib_db, with_ema session.query(User).filter(User.username == new_user_jwt["sub"]).one() -@pytest.mark.asyncio -async def test_get_current_user_user_is_inactive(session, setup_lib_db): +def test_get_current_user_user_is_inactive(session, setup_lib_db): mark_user_inactive(session, TEST_USER["username"]) - user_data = await get_current_user(None, TEST_USER_DECODED_JWT, session, None) + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) assert user_data is None -@pytest.mark.asyncio -async def test_get_current_user_set_active_roles(session, setup_lib_db): - user_data = await get_current_user(None, ADMIN_USER_DECODED_JWT, session, "admin") +def test_get_current_user_set_active_roles(session, setup_lib_db): + user_data = get_current_user(None, ADMIN_USER_DECODED_JWT, session, "admin") assert user_data.user.username == ADMIN_USER["username"] assert UserRole.admin in user_data.active_roles -@pytest.mark.asyncio -async def test_get_current_user_user_with_invalid_role_membership(session, setup_lib_db): +def test_get_current_user_user_with_invalid_role_membership(session, setup_lib_db): with pytest.raises(Exception) as exc_info: - await get_current_user(None, TEST_USER_DECODED_JWT, session, "admin") + get_current_user(None, TEST_USER_DECODED_JWT, session, "admin") assert "This user is not a member of the requested acting role." in str(exc_info.value.detail) -@pytest.mark.asyncio -async def test_get_current_user_user_extraneous_roles(session, setup_lib_db): - user_data = await get_current_user(None, TEST_USER_DECODED_JWT, session, "extra_role") +def test_get_current_user_user_extraneous_roles(session, setup_lib_db): + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, "extra_role") assert user_data.user.username == TEST_USER["username"] assert user_data.active_roles == [] -@pytest.mark.asyncio -async def test_get_current_user_concurrent_first_login_integrity_error_returns_existing_user(session, setup_lib_db): +def test_get_current_user_concurrent_first_login_integrity_error_returns_existing_user(session, setup_lib_db): """ Simulate two servers racing on first login: the commit raises IntegrityError because a concurrent request already inserted the row. The handler should roll back and return the @@ -162,7 +154,7 @@ def fake_commit(): session.commit = fake_commit with patch("mavedb.lib.authentication.fetch_orcid_user_email", return_value=None): - user_data = await get_current_user(None, new_user_jwt, session, None) + user_data = get_current_user(None, new_user_jwt, session, None) assert user_data is not None assert user_data.user.username == new_user_jwt["sub"] @@ -170,3 +162,48 @@ def fake_commit(): # Only one user record should exist in the database. users = session.query(User).filter(User.username == new_user_jwt["sub"]).all() assert len(users) == 1 + + +def _set_login_state(session, username, *, last_login, is_first_login=False): + user = session.query(User).filter(User.username == username).one() + user.last_login = last_login + user.is_first_login = is_first_login + session.commit() + return user + + +def test_get_current_user_skips_write_when_last_login_is_recent(session, setup_lib_db): + recent = datetime.now() - timedelta(minutes=1) + _set_login_state(session, TEST_USER["username"], last_login=recent) + + with patch.object(session, "commit", wraps=session.commit) as commit: + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.username == TEST_USER["username"] + commit.assert_not_called() + assert user_data.user.last_login == recent + + +def test_get_current_user_refreshes_stale_last_login(session, setup_lib_db): + stale = datetime.now() - LAST_LOGIN_REFRESH_INTERVAL - timedelta(minutes=1) + _set_login_state(session, TEST_USER["username"], last_login=stale) + + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.last_login > stale + + +def test_get_current_user_writes_when_last_login_is_missing(session, setup_lib_db): + _set_login_state(session, TEST_USER["username"], last_login=None) + + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.last_login is not None + + +def test_get_current_user_clears_first_login_flag_even_when_last_login_is_recent(session, setup_lib_db): + _set_login_state(session, TEST_USER["username"], last_login=datetime.now(), is_first_login=True) + + user_data = get_current_user(None, TEST_USER_DECODED_JWT, session, None) + + assert user_data.user.is_first_login is False diff --git a/tests/lib/test_calibration_control.py b/tests/lib/test_calibration_control.py new file mode 100644 index 000000000..4abc3ae6d --- /dev/null +++ b/tests/lib/test_calibration_control.py @@ -0,0 +1,446 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +import pandas as pd +from sqlalchemy.exc import IntegrityError + +from mavedb.lib.score_calibrations import build_calibration_controls, validate_calibration_controls_in_score_set +from mavedb.lib.validation.constants.general import ( + calibration_control_status_column_name, + calibration_variant_column_name, + hgvs_nt_column, + hgvs_pro_column, +) +from mavedb.lib.validation.dataframe.calibration import validate_and_standardize_calibration_controls_dataframe +from mavedb.lib.validation.exceptions import ValidationError +from mavedb.models.calibration_control import CalibrationControl +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.lib.mondo import get_generic_disease_term +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from mavedb.view_models.calibration_control import CalibrationControlCreate +from tests.helpers.constants import TEST_LICENSE, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, TEST_USER + + +def _make_calibration(session, score_set_id: int, user: User) -> ScoreCalibration: + """Persist a bare calibration on the given score set; controls need a parent calibration.""" + calibration = ScoreCalibration( + title="Calibration with controls", + score_set_id=score_set_id, + disease_term=get_generic_disease_term(session), + created_by=user, + modified_by=user, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + return calibration + + +def test_calibration_control_persists_and_links(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.id is not None + # Enum value round-trips through the varchar column. + assert control.clinical_status is CalibrationControlStatus.pathogenic + assert control.creation_date is not None + # Relationship resolves both directions via back_populates. + assert control.calibration is calibration + assert control.variant is variant + assert control in calibration.controls + + +def test_calibration_control_unique_per_calibration_and_variant(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + ) + session.commit() + + # A second control for the same (calibration, variant) pair violates the unique constraint, + # even with a different clinical status — a variant is one piece of evidence per calibration. + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.benign, + created_by=user, + modified_by=user, + ) + ) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + +def test_deleting_calibration_cascades_to_controls(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.benign, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + control_id = control.id + + # A control has no meaning apart from its calibration, so deleting the calibration removes it. + session.delete(calibration) + session.commit() + + assert session.get(CalibrationControl, control_id) is None + + +############################################################################## +# functional_classification_id placement (computed column_property) +############################################################################## + + +def _classification(calibration, label, functional_classification, score_range): + return ScoreCalibrationFunctionalClassification( + calibration=calibration, + label=label, + functional_classification=functional_classification, + range=score_range, + inclusive_lower_bound=True, + inclusive_upper_bound=False, + ) + + +def test_control_placement_prefers_classified_bin_over_not_specified(session, setup_lib_db_with_variant): + """A control in a not_specified-vs-classified overlap resolves to the classified bin. + + Overlapping ranges are only possible when one bin is 'not_specified' (see + ScoreCalibrationBase.ranges_do_not_overlap), and membership is built per-range, so the control's + variant lands in both bins. The placement must surface the clinically meaningful classified bin even + when the not_specified bin has the lower id — the tiebreak a plain id ordering would wrongly pick. + """ + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + # Create the not_specified bin first so it takes the lower id: this is the value a plain + # ``ORDER BY id`` would select, so asserting against the classified bin proves classified wins. + not_specified_bin = _classification( + calibration, "unclassified", FunctionalClassification.not_specified, [-2.0, 2.0] + ) + session.add(not_specified_bin) + session.commit() + + classified_bin = _classification(calibration, "abnormal", FunctionalClassification.abnormal, [0.0, 1.0]) + session.add(classified_bin) + session.commit() + + assert not_specified_bin.id < classified_bin.id + + # The variant sits in the overlap [0.0, 1.0], so it is a member of both bins. + not_specified_bin.variants = [variant] + classified_bin.variants = [variant] + session.commit() + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.functional_classification_id == classified_bin.id + + +def test_control_placement_falls_back_to_not_specified_when_no_classified_bin(session, setup_lib_db_with_variant): + """With no classified bin containing the variant, placement honestly reports the not_specified bin.""" + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + not_specified_bin = _classification( + calibration, "unclassified", FunctionalClassification.not_specified, [-2.0, 2.0] + ) + session.add(not_specified_bin) + session.commit() + not_specified_bin.variants = [variant] + session.commit() + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.benign, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.functional_classification_id == not_specified_bin.id + + +def test_control_placement_is_null_when_variant_in_no_bin(session, setup_lib_db_with_variant): + """A control whose variant falls under none of the calibration's classifications has no placement.""" + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + calibration = _make_calibration(session, variant.score_set_id, user) + + classified_bin = _classification(calibration, "abnormal", FunctionalClassification.abnormal, [0.0, 1.0]) + session.add(classified_bin) + session.commit() # bin exists but the variant is never added to its membership + + control = CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=user, + modified_by=user, + ) + session.add(control) + session.commit() + session.refresh(control) + + assert control.functional_classification_id is None + + +############################################################################## +# validate_calibration_controls_in_score_set (#751) +############################################################################## + + +def _variant_in_other_score_set(session, reference_score_set, user) -> Variant: + """Create a second score set (in the same experiment) with one variant, for cross-set tests.""" + scaffold = TEST_SEQ_SCORESET.copy() + scaffold.pop("target_genes") + other_score_set = ScoreSet( + **scaffold, + urn="urn:mavedb:00000002-a-1", + experiment_id=reference_score_set.experiment_id, + licence_id=TEST_LICENSE["id"], + ) + other_score_set.created_by = user + other_score_set.modified_by = user + session.add(other_score_set) + session.commit() + session.refresh(other_score_set) + + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{other_score_set.urn}#1", score_set_id=other_score_set.id) + session.add(variant) + session.commit() + session.refresh(variant) + return variant + + +@pytest.mark.parametrize("controls", [None, []]) +def test_validate_controls_is_a_no_op_when_absent(session, setup_lib_db_with_variant, controls): + variant = setup_lib_db_with_variant + # Must not raise for None or an empty list, and returns no URNs. + assert validate_calibration_controls_in_score_set(session, variant.score_set, controls) == [] + + +def test_validate_controls_accepts_variant_in_score_set(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + controls = [CalibrationControlCreate(variant_urn=variant.urn, clinical_status="pathogenic")] + + # Returns the validated URNs so persistence can reuse them without rebuilding the list. + assert validate_calibration_controls_in_score_set(session, variant.score_set, controls) == [variant.urn] + + +def test_validate_controls_rejects_nonexistent_variant(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + controls = [CalibrationControlCreate(variant_urn="urn:mavedb:99999999-x-9#1", clinical_status="benign")] + + with pytest.raises(ValidationError, match="do not belong to the calibration's score set"): + validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +def test_validate_controls_rejects_variant_from_another_score_set(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + other_variant = _variant_in_other_score_set(session, variant.score_set, user) + + controls = [CalibrationControlCreate(variant_urn=other_variant.urn, clinical_status="pathogenic")] + + with pytest.raises(ValidationError, match="do not belong to the calibration's score set"): + validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +def test_validate_controls_rejects_duplicate_variant_urns(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + controls = [ + CalibrationControlCreate(variant_urn=variant.urn, clinical_status="pathogenic"), + CalibrationControlCreate(variant_urn=variant.urn, clinical_status="benign"), + ] + + with pytest.raises(ValidationError, match="Duplicate control variant URNs detected"): + validate_calibration_controls_in_score_set(session, variant.score_set, controls) + + +############################################################################## +# build_calibration_controls (#753 lib persistence) +############################################################################## + + +@pytest.mark.parametrize("controls", [None, []]) +def test_build_calibration_controls_empty_for_absent_input(session, setup_lib_db_with_variant, controls): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + assert build_calibration_controls(session, variant.score_set, controls, user) == [] + + +def test_build_calibration_controls_constructs_rows(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + controls = [CalibrationControlCreate(variant_urn=variant.urn, clinical_status="benign")] + + built = build_calibration_controls(session, variant.score_set, controls, user) + + assert len(built) == 1 + assert built[0].variant is variant + assert built[0].clinical_status is CalibrationControlStatus.benign + assert built[0].created_by is user + assert built[0].modified_by is user + + +def test_build_calibration_controls_propagates_validation_error(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + controls = [CalibrationControlCreate(variant_urn="urn:mavedb:99999999-x-9#1", clinical_status="benign")] + + with pytest.raises(ValidationError, match="do not belong to the calibration's score set"): + build_calibration_controls(session, variant.score_set, controls, user) + + +############################################################################## +# validate_and_standardize_calibration_controls_dataframe (#753 CSV path) +############################################################################## + + +def _controls_df(variant_values, statuses): + return pd.DataFrame( + {calibration_variant_column_name: variant_values, calibration_control_status_column_name: statuses} + ) + + +def test_controls_csv_validates_and_converts_to_create_rows(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + # Status is case-insensitive. + df = _controls_df([variant.urn], ["Pathogenic"]) + + controls = validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + assert len(controls) == 1 + assert controls[0].variant_urn == variant.urn + assert controls[0].clinical_status is CalibrationControlStatus.pathogenic + + +def test_controls_csv_rejects_invalid_status(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + df = _controls_df([variant.urn], ["likely_pathogenic"]) + + with pytest.raises(ValidationError, match="Invalid clinical status"): + validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + +def test_controls_csv_rejects_variant_not_in_score_set(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + df = _controls_df(["urn:mavedb:99999999-x-9#1"], ["benign"]) + + with pytest.raises(ValidationError, match="do not exist in the score set"): + validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + +def test_controls_csv_rejects_missing_status_column(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + df = pd.DataFrame({calibration_variant_column_name: [variant.urn]}) + + with pytest.raises(ValidationError, match=calibration_control_status_column_name): + validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + +def test_controls_csv_accepts_hgvs_nt_index(session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + # hgvs_nt is unique within the score set, so it resolves 1:1 to the variant URN. + df = pd.DataFrame({hgvs_nt_column: [variant.hgvs_nt], calibration_control_status_column_name: ["benign"]}) + + controls = validate_and_standardize_calibration_controls_dataframe(session, variant.score_set, df) + + assert len(controls) == 1 + assert controls[0].variant_urn == variant.urn + assert controls[0].clinical_status is CalibrationControlStatus.benign + + +@pytest.mark.parametrize( + "index_column, shared_value, distinct_values", + [ + # hgvs_pro collision: two nucleotide variants collapsing to the same protein consequence + # (the expected case in a nucleotide-indexed score set). + (hgvs_pro_column, "p.Met1Val", ("c.1A>G", "c.3G>T")), + # hgvs_nt collision: not expected in practice, but the detection must still fire. + (hgvs_nt_column, "c.99A>G", ("p.Lys33Arg", "p.Lys33Gln")), + ], +) +def test_controls_csv_rejects_ambiguous_hgvs_index( + session, setup_lib_db_with_variant, index_column, shared_value, distinct_values +): + variant = setup_lib_db_with_variant + score_set = variant.score_set + + # Two variants that share the submitted index value but differ on the other HGVS field, so the + # index resolves to more than one variant in the score set. + for suffix, distinct_value in enumerate(distinct_values, start=2): + hgvs_nt = shared_value if index_column == hgvs_nt_column else distinct_value + hgvs_pro = shared_value if index_column == hgvs_pro_column else distinct_value + session.add( + Variant( + data=TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + ) + ) + session.commit() + + df = pd.DataFrame({index_column: [shared_value], calibration_control_status_column_name: ["pathogenic"]}) + + with pytest.raises(ValidationError, match="match more than one variant"): + validate_and_standardize_calibration_controls_dataframe(session, score_set, df) diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py new file mode 100644 index 000000000..4c3fe193c --- /dev/null +++ b/tests/lib/test_cat_vrs.py @@ -0,0 +1,570 @@ +# ruff: noqa: E402 +"""Tests for the on-the-fly Cat-VRS transit builder. + +The pure builder is unit-tested over transient ``MappingRecordAllele`` instances (no DB) — asserting +the mode, the member->defining relations, and the spec-pure ``CategoricalVariant`` shape for both +score-collapse modes, and the external cross-references carried in ``mappings``. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint +from ga4gh.cat_vrs.relations import Relation +from ga4gh.core.models import Relation as MappingRelation + +from mavedb.lib.allele_identity import AlleleDerivation +from mavedb.lib.cat_vrs import ( + _SPEC_EQUIVALENT, + _relation_concept, + CatVrsMode, + CatVrsRelation, + build_categorical_variant, + categorical_member_links, + is_projection_partner, + member_label, +) +from mavedb.lib.allele_annotations import AlleleCrossReferences, GnomadReference +from mavedb.models.allele import Allele +from mavedb.models.mapping_record_allele import MappingRecordAllele + +# A spec-valid 32-char VRS digest; the internal VRS digest is irrelevant to the builder, which keys +# member relations on the `vrs_digest` *column*, so one fixed valid value across members is fine. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A minimal but spec-valid post_mapped VRS Allele dict.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +_DEFAULT_POST_MAPPED = object() + + +def _link( + *, + level: str, + digest: str, + is_authoritative: bool, + post_mapped=_DEFAULT_POST_MAPPED, + projection_group=None, + caid=None, +) -> MappingRecordAllele: + """A transient (record, allele) link with its allele attached — no session needed. + + ``digest`` is the ``Allele.vrs_digest`` *column* (the key the builder uses for member relations), + deliberately distinct from the spec-valid VRS digest embedded in post_mapped. Pass + ``post_mapped=None`` to model an un-hydratable allele. ``projection_group`` pairs a c↔g projection + (the two links of one precise change share a value; the protein apex carries ``None``); the builder + uses it in projection mode to keep only the measured change's precise coordinate partner. ``caid`` is the + allele's ClinGen id, which the builder cross-references in ``mappings``. + """ + pm = _post_mapped() if post_mapped is _DEFAULT_POST_MAPPED else post_mapped + allele = Allele(level=level, vrs_digest=digest, post_mapped=pm, clingen_allele_id=caid) + return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele, projection_group=projection_group) + + +def _member_digests(links, *, include_convergent=True): + """The digests of the links a categorical variant draws its members from.""" + return {link.allele.vrs_digest for link in categorical_member_links(links, include_convergent=include_convergent)} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "focus_level, member_level, is_projection, expected", + [ + ("protein", "cdna", False, (CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE)), + ("protein", "genomic", True, (CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE)), + ("protein", "protein", False, (None, None)), + ("cdna", "protein", False, (CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION)), + ("cdna", "genomic", True, (CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION)), + ("genomic", "cdna", True, (CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION)), + ("cdna", "cdna", False, (CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT)), + ("genomic", "genomic", False, (CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT)), + ], +) +def test_member_label(focus_level, member_level, is_projection, expected): + """Every view labels a member from this one function, relative to the focus allele.""" + assert member_label(focus_level, member_level, is_projection=is_projection) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + "member_group, focus_group, expected", + [ + (0, 0, True), + (1, 0, False), + # Ungrouped either side can't be shown to pair: a pre-RT link, or a measured allele the fold-in missed. + (None, 0, False), + (0, None, False), + (None, None, False), + ], +) +def test_is_projection_partner(member_group, focus_group, expected): + assert is_projection_partner(member_group, focus_group=focus_group) is expected + + +@pytest.mark.unit +def test_mode_2_protein_measured_reverse_translation(): + """Protein measured: defining is the protein allele; both nt members `encodes` it (star model).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna", is_authoritative=False), + _link(level="genomic", digest="gen", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + + cv = transit.categorical_variant + assert isinstance(cv, CategoricalVariant) + # Every representation is a member, including the defining protein allele. + assert len(cv.members) == 3 + # Constraints come back as the `Constraint` union RootModel; unwrap to the concrete type. + constraint = cv.constraints[0].root + assert isinstance(constraint, DefiningAlleleConstraint) + # Relations on the constraint carry only the distinct kinds present (here: just `encodes`). + assert [str(r.primaryCoding.code.root) for r in constraint.relations] == ["encodes"] + + +@pytest.mark.unit +def test_mode_1_coding_measured_projection(): + """Coding measured: the projection_group partner is a coordinate representation; protein is a translation.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2") + assert transit is not None + + assert transit.mode == CatVrsMode.PROJECTION + + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of"} + + +@pytest.mark.unit +def test_mode_1_projection_includes_sibling_encoders(): + """Coding measured, full closure (default include_convergent=True): the reverse-translation fan on the + record (other encoders of the same protein consequence, in *different* projection groups) is kept as + members wearing the ``co_encodes`` relation, so the object is the full closure. The measured change's + own coordinate partner and protein consequence stay coordinate_representation_of / translation_of.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + # A sibling encoder of the same protein change — a distinct variant, different projection group. + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b") + assert transit is not None + + # members = defining cdna + gen partner + protein apex + the two cousins (full closure). + assert len(transit.categorical_variant.members) == 5 + # The distinct relation kinds surface on the constraint, including co_encodes. + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of", "co_encodes"} + + +@pytest.mark.unit +def test_mode_1_projection_narrow_object_drops_sibling_encoders(): + """Coding measured, narrow object (include_convergent=False, the VA subject): the synonymous cousins + in other projection groups are dropped — only the measured change's coordinate partner and protein + consequence remain. This pins the shape the VA-Spec path builds.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b-narrow", include_convergent=False) + assert transit is not None + + # Only the measured change's coordinate partner + the protein consequence; the cousins are excluded. + assert _member_digests(links, include_convergent=False) == {"cdna", "gen", "prot"} + # members = defining cdna + gen partner + protein apex (the two cousins are dropped). + assert len(transit.categorical_variant.members) == 3 + + +@pytest.mark.unit +def test_mode_2_reverse_translation_keeps_the_full_encoder_class(): + """Protein measured: every nt encoder stays, across projection groups — the full equivalence class + is what the protein claim ranges over (no sibling filtering in reverse-translation mode).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna_a", is_authoritative=False, projection_group=0), + _link(level="genomic", digest="gen_a", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="cdna_b", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="gen_b", is_authoritative=False, projection_group=1), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2c") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + # All four nt encoders stay, even in the narrow object; none dropped. + assert _member_digests(links, include_convergent=False) == {"prot", "cdna_a", "gen_a", "cdna_b", "gen_b"} + assert len(transit.categorical_variant.members) == 5 + + +@pytest.mark.unit +def test_no_authoritative_link_returns_none(): + """An unmapped variant (no authoritative link) has no defining allele to anchor on.""" + links = [_link(level="genomic", digest="gen", is_authoritative=False)] + assert build_categorical_variant(links, name="urn:mavedb:test#3") is None + + +@pytest.mark.unit +def test_empty_links_returns_none(): + assert build_categorical_variant([], name="urn:mavedb:test#4") is None + + +@pytest.mark.unit +@pytest.mark.parametrize( + "defining_level, expected_mode", + [ + ("protein", CatVrsMode.REVERSE_TRANSLATION), + ("cdna", CatVrsMode.PROJECTION), + ("genomic", CatVrsMode.PROJECTION), + ], +) +def test_mode_follows_defining_level(defining_level, expected_mode): + links = [_link(level=defining_level, digest="d", is_authoritative=True)] + transit = build_categorical_variant(links, name="urn:mavedb:test#5") + assert transit is not None + assert transit.mode == expected_mode + + +@pytest.mark.unit +def test_unhydratable_defining_allele_returns_none(): + """Defensive: an authoritative allele with no post_mapped can't anchor a Cat-VRS, so build → None + (a broken invariant in practice — the mapping job writes post_mapped on the authoritative allele).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True, post_mapped=None), + _link(level="cdna", digest="cdna", is_authoritative=False), + ] + assert build_categorical_variant(links, name="urn:mavedb:test#6") is None + + +@pytest.mark.unit +def test_unhydratable_member_allele_is_skipped(): + """Defensive: a member with no post_mapped is dropped; the build still succeeds on the rest.""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="good", is_authoritative=False), + _link(level="genomic", digest="bad", is_authoritative=False, post_mapped=None), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#7") + + assert transit is not None + # The un-hydratable genomic member is excluded from the members. + assert len(transit.categorical_variant.members) == 2 + + +# --- External cross-references: CategoricalVariant.mappings --- + +_CLINGEN = "https://reg.clinicalgenome.org/" +_GNOMAD = "https://gnomad.broadinstitute.org" +_CLINVAR = "https://www.ncbi.nlm.nih.gov/clinvar/variation/" + + +def _gnomad(db_identifier: str, db_version: str = "v4.1") -> AlleleCrossReferences: + return AlleleCrossReferences(gnomad=GnomadReference(db_identifier=db_identifier, db_version=db_version)) + + +def _clinvar(*variation_ids: str) -> AlleleCrossReferences: + """One ClinVar variation ID per live release; a repeated ID models one variation across releases.""" + return AlleleCrossReferences(clinvar_variation_ids=list(variation_ids)) + + +def _mappings(transit) -> dict[tuple[str, str], MappingRelation]: + """(system, code) -> relation, for the categorical variant's external cross-references.""" + return {(m.coding.system, m.coding.code.root): m.relation for m in transit.categorical_variant.mappings or []} + + +@pytest.mark.unit +def test_projection_mode_maps_the_measured_change_as_exact_and_other_variants_as_related(): + """The measured change and its coordinate twin are the categorical variant (exactMatch); the protein + consequence and convergent encodings are distinct variants (relatedMatch). The twins' shared CAID and + a ClinVar variation repeated across releases each collapse to one mapping.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0, caid="CA1"), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0, caid="CA1"), + _link(level="protein", digest="prot", is_authoritative=False, caid="PA1"), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1, caid="CA2"), + ] + references = { + "cdna": _clinvar("100", "100"), + "gen": _gnomad("1-100-A-G"), + "sibling_gen": _gnomad("1-101-C-T"), + } + + transit = build_categorical_variant(links, name="urn:mavedb:test#map1", cross_references=references) + + assert transit is not None + assert _mappings(transit) == { + (_CLINGEN, "CA1"): MappingRelation.EXACT_MATCH, + (_CLINVAR, "100"): MappingRelation.EXACT_MATCH, + (_GNOMAD, "1-100-A-G"): MappingRelation.EXACT_MATCH, + (_CLINGEN, "PA1"): MappingRelation.RELATED_MATCH, + (_CLINGEN, "CA2"): MappingRelation.RELATED_MATCH, + (_GNOMAD, "1-101-C-T"): MappingRelation.RELATED_MATCH, + } + + +@pytest.mark.unit +def test_narrow_object_omits_mappings_for_dropped_convergent_encodings(): + """Mappings follow the member set: the VA subject drops convergent encodings, so their records go too.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + ] + references = {"cdna": _gnomad("1-100-A-G"), "sibling_gen": _gnomad("1-101-C-T")} + + transit = build_categorical_variant( + links, name="urn:mavedb:test#map2", include_convergent=False, cross_references=references + ) + + assert transit is not None + assert _mappings(transit) == {(_GNOMAD, "1-100-A-G"): MappingRelation.EXACT_MATCH} + + +@pytest.mark.unit +def test_unfolded_measured_allele_treats_every_grouped_member_as_convergent(): + """A measured nt allele RT's fold-in missed has no group, so no member can be shown to be its projection. + Every grouped nt member is co_encodes rather than a coordinate representation, its records are + relatedMatch, and the VA subject keeps only the protein consequence.""" + links = [ + _link(level="genomic", digest="gen", is_authoritative=True, caid="CA1"), + _link(level="cdna", digest="twin_cdna", is_authoritative=False, projection_group=0, caid="CA1"), + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1, caid="CA2"), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1, caid="CA2"), + _link(level="protein", digest="prot", is_authoritative=False, caid="PA1"), + ] + references = {"gen": _clinvar("100"), "sibling_gen": _clinvar("200")} + + wide = build_categorical_variant(links, name="urn:mavedb:test#unfolded", cross_references=references) + narrow = build_categorical_variant( + links, name="urn:mavedb:test#unfolded", include_convergent=False, cross_references=references + ) + + assert wide is not None and narrow is not None + assert _mappings(wide) == { + (_CLINGEN, "CA1"): MappingRelation.EXACT_MATCH, + (_CLINVAR, "100"): MappingRelation.EXACT_MATCH, + (_CLINGEN, "CA2"): MappingRelation.RELATED_MATCH, + (_CLINVAR, "200"): MappingRelation.RELATED_MATCH, + (_CLINGEN, "PA1"): MappingRelation.RELATED_MATCH, + } + assert _member_digests(links, include_convergent=False) == {"gen", "prot"} + + +@pytest.mark.unit +def test_reverse_translation_maps_each_encoding_as_related(): + """Protein measured: the protein's own CAID is exact; every nt encoding is a distinct variant.""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True, caid="PA1"), + _link(level="cdna", digest="cdna_a", is_authoritative=False, projection_group=0, caid="CA1"), + _link(level="cdna", digest="cdna_b", is_authoritative=False, projection_group=1, caid="CA2"), + ] + references = {"cdna_a": _clinvar("100"), "cdna_b": _clinvar("200")} + + transit = build_categorical_variant(links, name="urn:mavedb:test#map3", cross_references=references) + + assert transit is not None + assert _mappings(transit) == { + (_CLINGEN, "PA1"): MappingRelation.EXACT_MATCH, + (_CLINGEN, "CA1"): MappingRelation.RELATED_MATCH, + (_CLINVAR, "100"): MappingRelation.RELATED_MATCH, + (_CLINGEN, "CA2"): MappingRelation.RELATED_MATCH, + (_CLINVAR, "200"): MappingRelation.RELATED_MATCH, + } + + +@pytest.mark.unit +def test_exact_match_wins_when_a_related_member_shares_the_identifier(): + """A record reached first through a related member is upgraded when an exact member also carries it.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="protein", digest="prot", is_authoritative=False), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + ] + references = {"prot": _clinvar("100"), "gen": _clinvar("100")} + + transit = build_categorical_variant(links, name="urn:mavedb:test#map4", cross_references=references) + + assert transit is not None + assert _mappings(transit) == {(_CLINVAR, "100"): MappingRelation.EXACT_MATCH} + + +@pytest.mark.unit +def test_mappings_are_resolvable_and_absent_without_identifiers(): + """Each coding carries an IRI; a variant with no external identifiers emits no ``mappings`` at all.""" + mapped = build_categorical_variant( + [_link(level="genomic", digest="gen", is_authoritative=True)], + name="urn:mavedb:test#map5", + cross_references={"gen": _gnomad("1-100-A-G")}, + ) + unmapped = build_categorical_variant( + [_link(level="genomic", digest="gen", is_authoritative=True)], name="urn:mavedb:test#map6" + ) + + assert mapped is not None and mapped.categorical_variant.mappings is not None + coding = mapped.categorical_variant.mappings[0].coding + assert [iri.root for iri in coding.iris or []] == [ + "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r4" + ] + assert unmapped is not None and unmapped.categorical_variant.mappings is None + + +@pytest.mark.unit +def test_gnomad_mapping_names_its_release(): + """The gnomAD coding carries the release it was matched in; its link is pinned by ``gnomad_variant_url``.""" + transit = build_categorical_variant( + [_link(level="genomic", digest="gen", is_authoritative=True)], + name="urn:mavedb:test#map7", + cross_references={"gen": _gnomad("1-100-A-G", "v2.1.1")}, + ) + + assert transit is not None and transit.categorical_variant.mappings is not None + coding = transit.categorical_variant.mappings[0].coding + assert coding.systemVersion == "v2.1.1" + assert [iri.root for iri in coding.iris or []] == [ + "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r2_1" + ] + + +@pytest.mark.unit +def test_mappings_do_not_depend_on_link_order(): + """The database returns links in no fixed order; the serialized mappings must not follow it.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0, caid="CA1"), + _link(level="protein", digest="prot", is_authoritative=False, caid="PA1"), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1, caid="CA2"), + ] + references = {"cdna": _clinvar("200", "100"), "sibling_gen": _gnomad("1-101-C-T")} + + forward = build_categorical_variant(links, name="urn:mavedb:test#map8", cross_references=references) + backward = build_categorical_variant( + list(reversed(links)), name="urn:mavedb:test#map8", cross_references=references + ) + + assert forward is not None and backward is not None + assert forward.categorical_variant.mappings == backward.categorical_variant.mappings + assert [(m.coding.system, m.coding.code.root) for m in forward.categorical_variant.mappings or []] == sorted( + _mappings(forward) + ) + + +@pytest.mark.unit +def test_member_links_put_the_defining_link_first_and_follow_the_narrow_selection(): + """The prefetch scope matches the built object: defining first, convergent encodings only when wide.""" + defining = _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0) + twin = _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0) + sibling = _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1) + links = [twin, sibling, defining] + + assert categorical_member_links(links) == [defining, twin, sibling] + assert categorical_member_links(links, include_convergent=False) == [defining, twin] + assert categorical_member_links([twin, sibling]) == [] + + +# --------------------------------------------------------------------------- +# Spec alignment +# +# MaveDB maintains its own relation vocabulary (Cat-VRS's `Relation` is an enum with members, so it +# cannot be subclassed, and inheriting would buy nothing — interop rides on `Coding.system`, not on +# Python enum identity). These tests are the drift guard that keeps the vocabulary honest: they fail +# when the spec gains a term MaveDB has not triaged, or when MaveDB coins a term without recording +# whether the spec already covers it. +# --------------------------------------------------------------------------- + + +def test_mapped_relations_emit_an_exact_match_to_the_spec_term(): + """A code with a spec equivalent must carry a machine-readable mapping to it. + + A matching code *string* is not enough: the two codings sit in different systems, so nothing but an + explicit ConceptMapping tells a consumer they are the same concept. + """ + concept = _relation_concept(CatVrsRelation.TRANSLATION_OF) + + assert concept.primaryCoding is not None + assert concept.primaryCoding.system == "https://mavedb.org/cat-vrs/relations" + assert concept.mappings is not None and len(concept.mappings) == 1 + + mapping = concept.mappings[0] + assert mapping.relation == MappingRelation.EXACT_MATCH + assert mapping.coding.code.root == Relation.TRANSLATION_OF.value + assert mapping.coding.system != concept.primaryCoding.system + + +@pytest.mark.parametrize( + "relation", + [CatVrsRelation.ENCODES, CatVrsRelation.CO_ENCODES, CatVrsRelation.COORDINATE_REPRESENTATION_OF], +) +def test_unmapped_relations_carry_no_mapping(relation): + """The absence of a mapping is the interoperable statement that no spec term covers this code. + + Emitting a `closeMatch` to something approximate would be worse than silence — a consumer would + resolve it and be wrong. See `_SPEC_EQUIVALENT` for why each of these three has no equivalent. + """ + concept = _relation_concept(relation) + + assert concept.primaryCoding is not None and concept.primaryCoding.code.root == relation.value + assert concept.mappings is None + + +def test_every_relation_is_either_mapped_or_a_declared_gap(): + """No MaveDB relation may exist without a triage decision recorded in `_SPEC_EQUIVALENT`. + + Coining a new code is fine; coining one *without deciding whether the spec already covers it* is the + drift this guards against. A new member fails here until it is either mapped or listed below. + """ + declared_gaps = { + CatVrsRelation.ENCODES, + CatVrsRelation.CO_ENCODES, + CatVrsRelation.COORDINATE_REPRESENTATION_OF, + } + + assert set(CatVrsRelation) == set(_SPEC_EQUIVALENT) | declared_gaps + + +def test_spec_relations_maveDB_deliberately_never_emits(): + """Fails when Cat-VRS publishes a relation MaveDB has not triaged. + + `liftover_to` is assembly-to-assembly, which MaveDB does not do. `transcribed_to` is genomic-> + transcript only and asserts transcription; MaveDB's c<->g relation is direction-neutral coordinate + equivalence that also holds for intronic/UTR-offset positions, so it is deliberately unmapped. + """ + emitted = {spec_term.value for spec_term in _SPEC_EQUIVALENT.values()} + never_emitted = {"liftover_to", "transcribed_to"} + + assert {relation.value for relation in Relation} == emitted | never_emitted diff --git a/tests/lib/test_clinical_controls.py b/tests/lib/test_clinical_controls.py new file mode 100644 index 000000000..194ff68bd --- /dev/null +++ b/tests/lib/test_clinical_controls.py @@ -0,0 +1,32 @@ +"""Unit tests for the clinical-controls lib helpers (``lib/clinical_controls.py``). + +The live-link query paths are exercised end-to-end by the score-set router tests; these pin the pure +``clinvar_version_sort_key``, which is the ``MM_YYYY`` release-ordering contract the UI parses in +parallel (``lib/clinical-controls.ts``) and so must stay stable. +""" + +from mavedb.lib.clinical_controls import clinvar_version_sort_key + + +def test_version_sort_key_orders_by_year_then_month(): + versions = ["01_2024", "12_2023", "06_2024", "03_2025"] + assert sorted(versions, key=clinvar_version_sort_key, reverse=True) == [ + "03_2025", + "06_2024", + "01_2024", + "12_2023", + ] + + +def test_version_sort_key_parses_month_and_year(): + # (year, month) — a later month within the same year sorts higher. + assert clinvar_version_sort_key("06_2024") == (2024, 6) + assert clinvar_version_sort_key("11_2024") > clinvar_version_sort_key("06_2024") + + +def test_version_sort_key_unparseable_sorts_to_bottom(): + # Malformed versions collapse to (0, 0) rather than raising, so they sink under any real release. + assert clinvar_version_sort_key("garbage") == (0, 0) + assert clinvar_version_sort_key("2024") == (0, 0) + assert clinvar_version_sort_key("13_20_2024") == (0, 0) + assert clinvar_version_sort_key("01_2020") > clinvar_version_sort_key("garbage") diff --git a/tests/lib/test_deprecation.py b/tests/lib/test_deprecation.py new file mode 100644 index 000000000..c15246b67 --- /dev/null +++ b/tests/lib/test_deprecation.py @@ -0,0 +1,48 @@ +# ruff: noqa: E402 +"""Tests for the shared RFC 8594 deprecation-header builder.""" + +import pytest + +# mavedb.lib.deprecation imports logging.context for record_deprecated_usage, which pulls in +# fastapi (an optional "server" extra) even though the functions tested here don't need it. +pytest.importorskip("fastapi") + +from mavedb.lib import deprecation +from mavedb.lib.deprecation import deprecation_headers + + +@pytest.mark.unit +def test_minimal_headers_are_deprecation_and_docs_link(): + headers = deprecation_headers() + assert headers["Deprecation"] == "true" + assert headers["Link"] == f'<{deprecation.DEPRECATION_DOCS_URL}>; rel="deprecation"' + assert "Sunset" not in headers # unset by default: no ratified removal date + assert "Warning" not in headers + + +@pytest.mark.unit +def test_successor_is_emitted_as_a_link_alongside_the_docs_link(): + headers = deprecation_headers(successor="/api/v1/variants/x", warning="moved") + link = headers["Link"] + assert '; rel="successor-version"' in link + assert 'rel="deprecation"' in link + assert headers["Warning"] == '299 - "moved"' + + +@pytest.mark.unit +def test_sunset_is_emitted_when_supplied(): + headers = deprecation_headers(sunset="Wed, 01 Jul 2026 00:00:00 GMT") + assert headers["Sunset"] == "Wed, 01 Jul 2026 00:00:00 GMT" + + +@pytest.mark.unit +def test_docs_link_can_be_omitted(): + headers = deprecation_headers(successor="/x", docs_url=None) + assert headers["Link"] == '; rel="successor-version"' + + +@pytest.mark.unit +def test_no_links_at_all_omits_the_link_header(): + headers = deprecation_headers(docs_url=None) + assert "Link" not in headers + assert headers == {"Deprecation": "true"} diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index 14dde9527..4c9dc9bc0 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -1,26 +1,29 @@ # ruff: noqa: E402 -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest - -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from sqlalchemy import select pyathena = pytest.importorskip("pyathena") fastapi = pytest.importorskip("fastapi") from mavedb.lib.gnomad import ( + GnomadLinkVerdict, allele_list_from_list_like_string, gnomad_identifier, gnomad_table_name, - link_gnomad_variants_to_mapped_variants, + gnomad_variant_url, + link_gnomad_variants_to_alleles, + normalize_caid, ) +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from tests.helpers.constants import ( TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, - TEST_MINIMAL_MAPPED_VARIANT, + TEST_MAVEDB_ATHENA_ROW, ) ### Tests for gnomad_identifier function ### @@ -77,6 +80,23 @@ def test_gnomad_table_name_raises_if_env_not_set(): gnomad_table_name() +### Tests for normalize_caid function ### + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("CA025094", "CA25094"), # the #722 example: gnomAD dump drops the leading zero + ("CA000123", "CA123"), # multiple leading zeros collapse + ("CA341478553", "CA341478553"), # already unpadded — unchanged + ("CA0", "CA0"), # keep the final digit even if it is a zero + ("not-a-caid", "not-a-caid"), # unrecognized input is passed through + ], +) +def test_normalize_caid(raw, expected): + assert normalize_caid(raw) == expected + + ### Tests for allele_list_from_list_like_string function ### @@ -118,217 +138,268 @@ def test_allele_list_from_list_like_string_invalid_format_not_list(): # If the package is working correctly, this function should work as expected. -### Tests for link_gnomad_variants_to_mapped_variants function ### +### Tests for link_gnomad_variants_to_alleles function ### -def _verify_annotation_status(session, mapped_variants, expected_version): - annotations = session.query(VariantAnnotationStatus).all() - assert len(annotations) == len(mapped_variants) +def _make_allele(session, caid, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele carrying a CAID.""" + allele = Allele(vrs_digest=vrs_digest, level=level, clingen_allele_id=caid) + session.add(allele) + session.commit() + session.refresh(allele) + return allele - for mapped_variant, annotation in zip(mapped_variants, annotations): - assert annotation.variant_id == mapped_variant.variant_id - assert annotation.annotation_type == "gnomad_allele_frequency" - assert annotation.version == expected_version +def _make_gnomad_row(**overrides): + """Build an Athena-style gnomAD row Mock from the canonical test row, applying overrides. -def test_links_new_gnomad_variant_to_mapped_variant( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() + Mirrors the ``mocked_gnomad_variant_row`` fixture but takes overrides so a test can construct a + batch of distinct rows (different CAIDs, loci) in one call. + """ + row = Mock() + for key, value in {**TEST_MAVEDB_ATHENA_ROW, **overrides}.items(): + setattr(row, key, value) + return row - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 - session.commit() - session.refresh(mapped_variant) +def _live_links_for(session, allele_id): + return session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.current, + ) + ).all() - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] +def _assert_gnomad_variant_matches(gnomad_variant, **overrides): + expected = TEST_GNOMAD_VARIANT.copy() + expected.pop("creation_date") + expected.pop("modification_date") + expected.update(overrides) + for attr, value in expected.items(): + assert getattr(gnomad_variant, attr) == value - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) +def test_links_new_gnomad_variant_to_allele(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} + session.commit() + + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant) -def test_can_link_gnomad_variants_with_none_type_faf_fields( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() + +def test_can_link_gnomad_variants_with_none_type_faf_fields(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max_gen_anc", None) mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max", None) with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - gnomad_variant_comparator["faf95_max"] = None - gnomad_variant_comparator["faf95_max_ancestry"] = None - - assert len(mapped_variant.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant, faf95_max=None, faf95_max_ancestry=None) -def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): +def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): gnomad_variant = GnomADVariant(**TEST_GNOMAD_VARIANT) - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) session.add(gnomad_variant) session.commit() + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") - - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] + # Reused the existing gnomAD variant rather than creating a second. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant_id == gnomad_variant.id - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) - -def test_adding_existing_gnomad_variant_with_same_version_does_not_result_in_duplication( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() +def test_re_running_unchanged_data_is_idempotent(session, mocked_gnomad_variant_row): + """Supersede only on change: a second run with identical data writes nothing — one live link, + no retired rows, so the valid-time history records no spurious boundary. The second run still + *reports* the allele (verdict UNCHANGED), so the caller can mark it preexisting without re-querying + link state.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 - - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } + session.commit() + # Second run sees the live link already points to this gnomAD variant → unchanged, no DB write. + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.UNCHANGED + } session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") + # One link, still live, never retired — the re-run did not churn the history. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 1 + assert all_links[0].valid_to is None + assert len(session.scalars(select(GnomADVariant)).all()) == 1 - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] - _verify_annotation_status(session, [mapped_variant, mapped_variant], TEST_GNOMAD_DATA_VERSION) +def test_version_bump_supersedes_to_single_live_link(session, mocked_gnomad_variant_row): + """A new gnomAD version retires the prior link and installs the new one — exactly one live link + per allele (not one per version), with the old version preserved as a retired row.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v1.old"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } + session.commit() -def test_links_multiple_rows_and_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v2.new"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } + session.commit() - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_version == "v2.new" + # Old-version link retired, not deleted; both gnomAD variant rows persist. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 2 + assert len([link for link in all_links if link.valid_to is not None]) == 1 + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + + +def test_same_version_different_identifier_supersedes_newest_wins(session, mocked_gnomad_variant_row): + """A CAID re-resolving to a different identifier within the same version is an anomaly: log and + supersede newest-wins rather than raise — one odd allele must not abort the batch.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + # Prior live link at the current version, but to a different identifier than the row resolves to. + stale = GnomADVariant( + db_name="gnomAD", + db_identifier="9-99999-C-T", + db_version=TEST_GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(stale) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=stale.id)) session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 2 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - - assert len(mapped_variant1.gnomad_variants) == 1 - assert len(mapped_variant2.gnomad_variants) == 1 - for mv in [mapped_variant1, mapped_variant2]: - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_identifier != "9-99999-C-T" # newest wins - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) +def test_links_one_gnomad_variant_to_multiple_alleles_sharing_a_caid(session, mocked_gnomad_variant_row): + """A CAID shared by multiple alleles (cross-score-set dedup) fans the gnomAD variant to each.""" + allele1 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + allele2 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-2", level="cdna") -def test_returns_zero_when_no_mapped_variants(session, mocked_gnomad_variant_row): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 0 + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele1.id: GnomadLinkVerdict.CREATED, allele2.id: GnomadLinkVerdict.CREATED} + session.commit() - _verify_annotation_status(session, [], TEST_GNOMAD_DATA_VERSION) + for allele in (allele1, allele2): + assert len(_live_links_for(session, allele.id)) == 1 + # Both links point at the single get-or-created gnomAD variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 -def test_only_current_flag_filters_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() +def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gnomad_variant_row): + """The gnomAD dump records CAIDs without leading zeros (#722). An allele stored with the + zero-padded CAID must still match the dump's stripped form across the join.""" + allele = _make_allele(session, "CA025094", vrs_digest="vrs-1") + mocked_gnomad_variant_row.caid = "CA25094" # dump form: leading zero stripped with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - - assert len(mapped_variant1.gnomad_variants) == 0 - assert len(mapped_variant2.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant2.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + assert len(_live_links_for(session, allele.id)) == 1 - _verify_annotation_status(session, [mapped_variant2], TEST_GNOMAD_DATA_VERSION) +def test_links_a_batch_of_rows_in_one_pass(session): + """Multiple rows in one call are resolved by a single batched allele lookup, then grouped by + normalized CAID. Exercises everything the batching must preserve at once: distinct CAIDs each + reach their own allele, a zero-padded stored CAID still matches a dump-stripped row (#722), and a + CAID shared across alleles fans its variant out to each — all without a per-row query.""" + # Row A: allele stored zero-padded, dump delivers the stripped form. + allele_a = _make_allele(session, "CA025094", vrs_digest="vrs-a") + row_a = _make_gnomad_row(caid="CA25094") + row_a.__setattr__("locus.position", "87961093") # identifier 10-87961093-A-G -def test_only_current_flag_is_false_operates_on_all_variants( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() + # Row B: a different CAID at a different locus, shared by two alleles (cross-score-set dedup). + allele_b1 = _make_allele(session, "CA341478553", vrs_digest="vrs-b1") + allele_b2 = _make_allele(session, "CA341478553", vrs_digest="vrs-b2", level="cdna") + row_b = _make_gnomad_row(caid="CA341478553") + row_b.__setattr__("locus.position", "87961094") # identifier 10-87961094-A-G with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row], False) - assert result == 2 + result = link_gnomad_variants_to_alleles(session, [row_a, row_b]) + assert result == { + allele_a.id: GnomadLinkVerdict.CREATED, + allele_b1.id: GnomadLinkVerdict.CREATED, + allele_b2.id: GnomadLinkVerdict.CREATED, + } session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - for mv in [mapped_variant1, mapped_variant2]: - assert len(mv.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + # Each allele ends with exactly one live link. + for allele in (allele_a, allele_b1, allele_b2): + assert len(_live_links_for(session, allele.id)) == 1 + + # Two distinct gnomAD variants (one per row's identifier); B's two alleles share row B's variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + b1_variant = _live_links_for(session, allele_b1.id)[0].gnomad_variant_id + b2_variant = _live_links_for(session, allele_b2.id)[0].gnomad_variant_id + a_variant = _live_links_for(session, allele_a.id)[0].gnomad_variant_id + assert b1_variant == b2_variant + assert a_variant != b1_variant + + +def test_returns_empty_map_when_no_alleles_match(session, mocked_gnomad_variant_row): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {} + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 + # No gnomAD variant is created when nothing matches the CAID. + assert len(session.scalars(select(GnomADVariant)).all()) == 0 + + +@pytest.mark.unit +@pytest.mark.parametrize( + "db_version,expected_url", + [ + ("v4.1", "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r4"), + ("v3.1.2", "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r3"), + ("v2.1.1", "https://gnomad.broadinstitute.org/variant/1-100-A-G?dataset=gnomad_r2_1"), + ("custom", "https://gnomad.broadinstitute.org/variant/1-100-A-G"), + ], +) +def test_gnomad_variant_url_pins_the_dataset_of_a_known_release(db_version, expected_url): + assert gnomad_variant_url("1-100-A-G", db_version) == expected_url + - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) +@pytest.mark.unit +def test_gnomad_variant_url_encodes_the_identifier(): + assert gnomad_variant_url("1-100-A/G", "v4.1") == ( + "https://gnomad.broadinstitute.org/variant/1-100-A%2FG?dataset=gnomad_r4" + ) diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py new file mode 100644 index 000000000..47687c6a1 --- /dev/null +++ b/tests/lib/test_hgvs.py @@ -0,0 +1,216 @@ +import pytest + +from mavedb.lib.hgvs import ( + SequenceBlock, + extract_accession, + is_cis_phased_hgvs, + join_cis_phased_hgvs, + parse_simple_nucleotide_substitution, + parse_simple_protein_substitution, + parse_simple_substitution, + split_cis_phased_hgvs, + strip_protein_prediction_parens, +) + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # auto-detects the level from the disjoint coordinate prefix + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + # non-placeable in either grammar + ("c.122-6T>A", None), + ("c.[197A>G;472T>C]", None), + (None, None), + ], +) +def test_parse_simple_substitution(hgvs, expected): + assert parse_simple_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified and bare coding substitutions + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("c.5A>G", SequenceBlock(position=5, ref="A", alt="G")), + # genomic and non-coding levels parse the same way + ("NC_000001.11:g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + ("n.42C>T", SequenceBlock(position=42, ref="C", alt="T")), + # lowercase nucleotides are tolerated + ("c.5a>g", SequenceBlock(position=5, ref="a", alt="g")), + # non-placeable: UTR/intron positions, multivariant, indels, non-substitutions, empty + ("c.*123A>G", None), + ("c.-12A>G", None), + ("c.12+3A>G", None), + ("NM_000546.6:c.[197A>G;472T>C]", None), + ("c.76_78del", None), + ("p.Ala406Thr", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_nucleotide_substitution(hgvs, expected): + assert parse_simple_nucleotide_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified, bare, and prediction-wrapped substitutions + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("NP_000537.3:p.(Ala406Thr)", SequenceBlock(position=406, ref="Ala", alt="Thr")), + # synonymous, stop, and deletion tokens are preserved as the raw alt + ("p.Ala406=", SequenceBlock(position=406, ref="Ala", alt="=")), + ("p.Tyr745*", SequenceBlock(position=745, ref="Tyr", alt="*")), + ("p.Ala406-", SequenceBlock(position=406, ref="Ala", alt="-")), + # non-placeable: multivariant, frameshift, single-letter codes, empty + ("p.[Ala406Thr;Gly12Cys]", None), + ("p.Arg97fs", None), + ("p.A406T", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_protein_substitution(hgvs, expected): + assert parse_simple_protein_substitution(hgvs) == expected + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NP_000456.2:p.(Ala222Val)", "NP_000456.2:p.Ala222Val"), + ("NP_000456.2:p.(Tyr745Ter)", "NP_000456.2:p.Tyr745Ter"), + ("NP_000456.2:p.(Ala222_Val225del)", "NP_000456.2:p.Ala222_Val225del"), + # no prediction parens -> unchanged + ("NP_000456.2:p.Ala222Val", "NP_000456.2:p.Ala222Val"), + ("NM_000001.1:c.5A>G", "NM_000001.1:c.5A>G"), + ], +) +def test_strip_protein_prediction_parens(hgvs, expected): + assert strip_protein_prediction_parens(hgvs) == expected + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NC_000001.11:g.1000A>G", "NC_000001.11"), + ("NM_000001.1:c.5A>G", "NM_000001.1"), + ("NC_000001.11:g.[1000A>G;1002T>C]", "NC_000001.11"), + ("no-colon-here", ""), + ("", ""), + ], +) +def test_extract_accession(hgvs, expected): + assert extract_accession(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NC_000016.10:g.[1314031A>T;1320511T>C]", True), + ("NM_000001.1:c.[1A>G;3T>C]", True), + ("NC_000016.10:g.[1314031A>T]", True), # single-member allele list still fails the forward parser + ("NC_000001.11:g.1000A>G", False), + ("NM_000546.6:c.1216G>A", False), + ("NM_000001.1:c.101_102[4]", False), # tandem-repeat brackets are not an allele list + ("g.[1000A>G;1002T>C]", False), # accession-less: not the shape RT feeds the engine + ], +) +def test_is_cis_phased_hgvs(hgvs, expected): + assert is_cis_phased_hgvs(hgvs) is expected + + +def test_split_cis_phased_hgvs_passes_through_single_variant(): + assert split_cis_phased_hgvs("NC_000001.11:g.1000A>G") == ["NC_000001.11:g.1000A>G"] + + +def test_split_cis_phased_hgvs_splits_bracketed_genomic_expression(): + # Non-adjacent codon components are emitted as one bracketed genomic expression; each + # component must regain the accession and prefix to be VRS-translatable on its own. + assert split_cis_phased_hgvs("NC_000001.11:g.[1000A>G;1002T>C]") == [ + "NC_000001.11:g.1000A>G", + "NC_000001.11:g.1002T>C", + ] + + +def test_split_cis_phased_hgvs_handles_three_components_and_coding_prefix(): + assert split_cis_phased_hgvs("NM_000001.1:c.[1A>G;3T>C;5G>A]") == [ + "NM_000001.1:c.1A>G", + "NM_000001.1:c.3T>C", + "NM_000001.1:c.5G>A", + ] + + +def test_join_cis_phased_hgvs_passes_through_single_component(): + assert join_cis_phased_hgvs(["NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.1000A>G" + + +def test_join_cis_phased_hgvs_combines_genomic_components(): + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +@pytest.mark.parametrize( + "expression", + [ + "NC_000001.11:g.[1000A>G;1002T>C]", + "NM_000001.1:c.[1A>G;3T>C;5G>A]", + ], +) +def test_join_is_inverse_of_split(expression): + assert join_cis_phased_hgvs(split_cis_phased_hgvs(expression)) == expression + + +def test_join_cis_phased_hgvs_returns_none_for_empty(): + assert join_cis_phased_hgvs([]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_accessions(): + # Members on different sequences are not one cis-phased block. + assert join_cis_phased_hgvs(["NC_000001.11:g.1A>G", "NC_000002.12:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_coordinate_prefixes(): + assert join_cis_phased_hgvs(["NM_000001.1:c.1A>G", "NM_000001.1:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_component_without_accession(): + assert join_cis_phased_hgvs(["g.1A>G", "g.2T>C"]) is None + + +def test_split_cis_phased_hgvs_passes_through_bracketed_without_accession(): + # Bracketed but accession-less input is not a cis-phased multivariant we can qualify; it must + # degrade to a single-element list rather than raising on the missing ":". + assert split_cis_phased_hgvs("g.[1000A>G;1002T>C]") == ["g.[1000A>G;1002T>C]"] + + +def test_join_cis_phased_hgvs_orders_components_by_position(): + # Out-of-order members are emitted in coordinate order. + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +def test_join_cis_phased_hgvs_is_order_independent(): + # The same set of members yields the same string regardless of input ordering (the VRS block + # digest is order-independent; the exported HGVS string must be too). + forward = join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) + reverse = join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) + assert forward == reverse + + +def test_join_cis_phased_hgvs_orders_protein_components_by_position(): + # The first integer is the position for protein forms too (Arg123Gly), not just genomic. + assert ( + join_cis_phased_hgvs(["NP_000001.1:p.Arg223Gly", "NP_000001.1:p.Ala12Val"]) + == "NP_000001.1:p.[Ala12Val;Arg223Gly]" + ) diff --git a/tests/lib/test_identifiers.py b/tests/lib/test_identifiers.py new file mode 100644 index 000000000..cc378223b --- /dev/null +++ b/tests/lib/test_identifiers.py @@ -0,0 +1,98 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +import requests + +from mavedb.lib.identifiers import find_generic_article +from tests.helpers.constants import TEST_BIORXIV_IDENTIFIER, TEST_PUBMED_IDENTIFIER + +PUBMED_EFETCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + + +def mock_pubmed_success(requests_mock, identifier): + requests_mock.post( + PUBMED_EFETCH_URL, + text=f""" + + + + {identifier} +
+ + test + + + 1999 + + + + + test + +
+ + + + test + + + + + """, + ) + + +def medrxiv_url(identifier): + return f"https://api.biorxiv.org/details/medrxiv/10.1101/{identifier}/na/json" + + +def biorxiv_url(identifier): + return f"https://api.biorxiv.org/details/biorxiv/10.1101/{identifier}/na/json" + + +@pytest.mark.asyncio +async def test_find_generic_article_drops_candidate_with_malformed_response(session, requests_mock): + """ + A PMID like TEST_PUBMED_IDENTIFIER is also a plausible (8 digit) legacy medRxiv + identifier, so find_generic_article fans out to both databases. If medRxiv responds + with a body that can't be parsed as JSON, that candidate should be dropped (and logged) + rather than failing the whole lookup when PubMed already has a match. + """ + mock_pubmed_success(requests_mock, TEST_PUBMED_IDENTIFIER) + requests_mock.get(medrxiv_url(TEST_PUBMED_IDENTIFIER), text="not json") + + matches = await find_generic_article(session, TEST_PUBMED_IDENTIFIER) + + assert matches["PubMed"] is not None + assert matches.get("medRxiv") is None + + +@pytest.mark.asyncio +async def test_find_generic_article_drops_candidate_on_request_exception(session, requests_mock): + """ + The same fan-out should also tolerate a candidate database being entirely unreachable + (timeouts, connection errors, bad status codes), not just malformed JSON bodies. + """ + mock_pubmed_success(requests_mock, TEST_PUBMED_IDENTIFIER) + requests_mock.get(medrxiv_url(TEST_PUBMED_IDENTIFIER), exc=requests.exceptions.ConnectTimeout) + + matches = await find_generic_article(session, TEST_PUBMED_IDENTIFIER) + + assert matches["PubMed"] is not None + assert matches.get("medRxiv") is None + + +@pytest.mark.asyncio +async def test_find_generic_article_reraises_error_when_no_candidate_succeeds(session, requests_mock): + """ + When every candidate database fails and none produces a match, the underlying error + should propagate instead of being silently reported as "not found", so callers can + still surface it as a gateway error (eg. a 504 on a Rxiv timeout). + """ + requests_mock.get(biorxiv_url(TEST_BIORXIV_IDENTIFIER), exc=requests.exceptions.ConnectTimeout) + + with pytest.raises(requests.exceptions.ConnectTimeout): + await find_generic_article(session, TEST_BIORXIV_IDENTIFIER) diff --git a/tests/lib/test_score_calibrations.py b/tests/lib/test_score_calibrations.py index 3b080e90d..b941cc089 100644 --- a/tests/lib/test_score_calibrations.py +++ b/tests/lib/test_score_calibrations.py @@ -1820,9 +1820,9 @@ def test_variants_for_functional_classification_raises_error_when_index_column_n variant_classes = pd.DataFrame( { "some_other_column": [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", ], calibration_class_column_name: [ "pathogenic", @@ -1851,9 +1851,9 @@ def test_variants_for_functional_classification_raises_error_when_index_column_n pd.DataFrame( { calibration_variant_column_name: [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", ], calibration_class_column_name: [ "pathogenic", @@ -1910,21 +1910,21 @@ def test_variants_for_functional_classification_python_filtering_with_valid_vari variant_1 = Variant( data={"score_data": {"score": 0.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", hgvs_nt="NC_000001.11:g.1000A>T", hgvs_pro="NP_000000.1:p.Lys100Asn", ) variant_2 = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", hgvs_nt="NC_000001.11:g.1001G>C", hgvs_pro="NP_000000.1:p.Gly101Arg", ) variant_3 = Variant( data={"score_data": {"score": 2.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-3", + urn=f"{VALID_SCORE_SET_URN}#3", hgvs_nt="NC_000001.11:g.1002T>A", hgvs_pro="NP_000000.1:p.Ser102Thr", ) @@ -1967,14 +1967,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_without_score_data = Variant( data={"other_data": {"value": 1.0}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_without_score_data, variant_with_score]) @@ -2013,14 +2013,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_invalid_score_data = Variant( data={"score_data": "not_a_dict"}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_invalid_score_data, variant_with_score]) @@ -2058,14 +2058,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_none_score = Variant( data={"score_data": {"score": None}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_none_score, variant_with_score]) @@ -2103,14 +2103,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ variant_string_score = Variant( data={"score_data": {"score": "not_a_number"}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_string_score, variant_with_score]) @@ -2146,14 +2146,14 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ ): # Create variant with non-dict data variant_invalid_data = Variant( - data="not_a_dict", score_set_id=setup_lib_db_with_score_set.id, urn="urn:mavedb:variant-1" + data="not_a_dict", score_set_id=setup_lib_db_with_score_set.id, urn=f"{VALID_SCORE_SET_URN}#1" ) # Create variant with valid score variant_with_score = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-2", + urn=f"{VALID_SCORE_SET_URN}#2", ) session.add_all([variant_invalid_data, variant_with_score]) @@ -2191,11 +2191,11 @@ def test_variants_for_functional_classification_python_filtering_skips_variants_ pd.DataFrame( { calibration_variant_column_name: [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", - "urn:mavedb:variant-4", - "urn:mavedb:variant-5", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", + f"{VALID_SCORE_SET_URN}#4", + f"{VALID_SCORE_SET_URN}#5", ], calibration_class_column_name: [ "pathogenic", @@ -2280,7 +2280,7 @@ def test_variants_for_functional_classification_filters_by_conditions( variant = Variant( data={"score_data": {"score": score}}, score_set_id=setup_lib_db_with_score_set.id, - urn=f"urn:mavedb:variant-{i}", + urn=f"{VALID_SCORE_SET_URN}#{i}", hgvs_nt=hgvs_nts[i - 1], hgvs_pro=hgvs_pros[i - 1], ) @@ -2325,9 +2325,9 @@ def test_variants_for_functional_classification_filters_by_conditions( pd.DataFrame( { calibration_variant_column_name: [ - "urn:mavedb:variant-1", - "urn:mavedb:variant-2", - "urn:mavedb:variant-3", + f"{VALID_SCORE_SET_URN}#1", + f"{VALID_SCORE_SET_URN}#2", + f"{VALID_SCORE_SET_URN}#3", ], calibration_class_column_name: [ "benign", @@ -2385,7 +2385,7 @@ def test_variants_for_functional_classification_sql_fallback_on_exception( variant = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-1", + urn=f"{VALID_SCORE_SET_URN}#1", hgvs_nt="NC_000001.11:g.1000A>T", hgvs_pro="NP_000000.1:p.Lys100Asn", ) @@ -2441,7 +2441,7 @@ def test_variants_for_functional_classification_sql_with_infinite_bound( variant = Variant( data={"score_data": {"score": score}}, score_set_id=setup_lib_db_with_score_set.id, - urn=f"urn:mavedb:variant-{i}", + urn=f"{VALID_SCORE_SET_URN}#{i}", ) variants.append(variant) @@ -2495,7 +2495,7 @@ def test_variants_for_functional_classification_sql_with_exclusive_bounds( variant = Variant( data={"score_data": {"score": score}}, score_set_id=setup_lib_db_with_score_set.id, - urn=f"urn:mavedb:variant-{i}", + urn=f"{VALID_SCORE_SET_URN}#{i}", ) variants.append(variant) @@ -2555,10 +2555,10 @@ def test_variants_for_functional_classification_only_returns_variants_from_corre variant_in_target_set = Variant( data={"score_data": {"score": 1.5}}, score_set_id=setup_lib_db_with_score_set.id, - urn="urn:mavedb:variant-target", + urn=f"{VALID_SCORE_SET_URN}#1", ) variant_in_other_set = Variant( - data={"score_data": {"score": 1.5}}, score_set_id=other_score_set.id, urn="urn:mavedb:variant-other" + data={"score_data": {"score": 1.5}}, score_set_id=other_score_set.id, urn=f"{other_score_set.urn}#1" ) session.add_all([variant_in_target_set, variant_in_other_set]) @@ -2581,7 +2581,7 @@ def test_variants_for_functional_classification_only_returns_variants_from_corre # Should only return variant from the target score set assert len(result) == 1 assert result[0].score_set_id == setup_lib_db_with_score_set.id - assert result[0].urn == "urn:mavedb:variant-target" + assert result[0].urn == f"{VALID_SCORE_SET_URN}#1" ################################################################################ diff --git a/tests/lib/test_score_distribution.py b/tests/lib/test_score_distribution.py new file mode 100644 index 000000000..0f26dd5cb --- /dev/null +++ b/tests/lib/test_score_distribution.py @@ -0,0 +1,78 @@ +import math + +import pytest + +from mavedb.lib.score_distribution import ( + SCORE_DISTRIBUTION_BIN_COUNT, + SCORE_DISTRIBUTION_VERSION, + summarize_scores, + summarize_variant_data, +) + + +@pytest.mark.parametrize("scores", [[], [None], [None, math.nan, math.inf]]) +def test_summarize_scores_without_numeric_scores_returns_none(scores): + assert summarize_scores(scores) is None + + +def test_summarize_scores_shape(): + summary = summarize_scores([0.0, 1.0]) + + assert summary is not None + assert summary["version"] == SCORE_DISTRIBUTION_VERSION + assert summary["min"] == 0.0 + assert summary["max"] == 1.0 + assert len(summary["counts"]) == SCORE_DISTRIBUTION_BIN_COUNT + assert summary["null_count"] == 0 + + +def test_summarize_scores_puts_min_in_first_bin_and_max_in_last_bin(): + summary = summarize_scores([-2.0, 0.0, 4.0]) + + assert summary is not None + assert summary["counts"][0] == 1 + assert summary["counts"][-1] == 1 + assert sum(summary["counts"]) == 3 + + +def test_summarize_scores_bins_by_equal_width(): + # Width is 1.0 over [0, 24]; 11.5 falls in bin 11 and 12.0 starts bin 12. + summary = summarize_scores([0.0, 11.5, 12.0, 24.0]) + + assert summary is not None + assert summary["counts"][11] == 1 + assert summary["counts"][12] == 1 + + +def test_summarize_scores_single_repeated_value_lands_in_first_bin(): + summary = summarize_scores([3.5, 3.5, 3.5]) + + assert summary is not None + assert summary["min"] == summary["max"] == 3.5 + assert summary["counts"][0] == 3 + assert sum(summary["counts"]) == 3 + + +def test_summarize_scores_counts_nulls_separately(): + summary = summarize_scores([1.0, None, 2.0, math.nan, 3.0]) + + assert summary is not None + assert summary["null_count"] == 2 + assert sum(summary["counts"]) == 3 + assert (summary["min"], summary["max"]) == (1.0, 3.0) + + +def test_summarize_variant_data_reads_canonical_score(): + variant_data = [ + {"score_data": {"score": 1.0}}, + {"score_data": {"score": "2.0"}}, + {"score_data": {"score": None}}, + {"score_data": {}}, + None, + ] + + summary = summarize_variant_data(variant_data) + + assert summary is not None + assert sum(summary["counts"]) == 2 + assert summary["null_count"] == 3 diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py new file mode 100644 index 000000000..9a516bc3e --- /dev/null +++ b/tests/lib/test_score_set_variants.py @@ -0,0 +1,413 @@ +# ruff: noqa: E402 +"""Integration tests for the lean whole-set view assembly (``lib/score_set_variants.py``). + +These pin the one-row-per-variant assembly: the submitted HGVS come off the variant, the mapped +assay-level HGVS off the live mapping record, the digest/ClinGen id/VEP consequence off the +authoritative allele; each HGVS string always rides even when it is not a placeable simple +substitution; unmapped variants are retained with null mapped fields; ``as_of`` reconstructs the +annotation layer while the immutable submitted HGVS and score stay put; and the set is ordered by +variant number. They also pin the query's plan shape: per-variant lookups stay keyed index probes. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.score_set_variants import HgvsField, MappedTriple, get_lean_score_set_variants +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.query_plan import captured_statements, unkeyed_scans + +# Deterministic windows far from the transaction clock (mirrors tests/lib/test_alleles.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _variant(session, score_set, suffix, *, score=None, hgvs_nt=None, hgvs_pro=None, hgvs_splice=None): + data = {"score_data": {"score": score}} if score is not None else TEST_MINIMAL_VARIANT["data"] + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data, + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + hgvs_splice=hgvs_splice, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_c=None, hgvs_g=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped={"type": "Allele"}, + clingen_allele_id=clingen_allele_id, + hgvs_c=hgvs_c, + hgvs_g=hgvs_g, + hgvs_p=hgvs_p, + ) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, valid_from=None): + record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + valid_from=valid_from, + ) + session.add(record) + session.commit() + return record + + +def _group(session, record, *, hgvs_c=None, hgvs_g=None, hgvs_p=None, valid_from=None): + group = ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=hgvs_c, + hgvs_g=hgvs_g, + hgvs_p=hgvs_p, + valid_from=valid_from, + ) + session.add(group) + session.commit() + return group + + +def _link(session, record, allele, *, is_authoritative=True, group=None, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=group.id if group is not None else None, + valid_from=valid_from, + ) + session.add(link) + session.commit() + return link + + +def _consequence(session, allele, value): + row = VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + session.add(row) + session.commit() + return row + + +@pytest.mark.integration +def test_nucleotide_assay_measured_at_cdna(session, setup_lib_db_with_score_set): + """Coding assay: submitted nt+pro from the variant; the mapped triple carries the measured cdna slot + (from the record) and the measured group's genomic and protein strings; digest/ClinGen id/consequence off + the authoritative allele. Each parseable string gets its block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, is_authoritative=False, group=measured_group) + _link(session, record, protein, is_authoritative=False, group=measured_group) + _consequence(session, measured, "missense_variant") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == -2.3 + assert record_out.consequence == "missense_variant" + assert record_out.clingen_allele_id == "CA123" + assert record_out.assay_level_digest == "cdna-digest" # the measured (authoritative) allele + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.hgvs_pro == HgvsField(hgvs="p.Thr1Ser", position=1, ref="Thr", alt="Ser") + assert record_out.hgvs_splice is None + # assayLevel names the measured slot; the triple fills all three levels. + assert record_out.assay_level == "cdna" + assert record_out.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + +@pytest.mark.integration +def test_nucleotide_assay_measured_at_genomic(session, setup_lib_db_with_score_set): + """Genomic assay: the measured slot is genomic (from the record); the cdna slot — the level-invariant + search key — comes off the measured group even though the assay was measured at genomic.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA9") + cdna = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, cdna, is_authoritative=False, group=measured_group) + _link(session, record, protein, is_authoritative=False, group=measured_group) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "genomic" + assert record_out.assay_level_digest == "gen-digest" + # Measured slot is genomic (from the record); cdna (search key) comes off the measured group. + assert record_out.mapped == MappedTriple( + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + +@pytest.mark.integration +def test_protein_assay_serves_only_the_protein_slot(session, setup_lib_db_with_score_set): + """Protein assay: the measured slot *is* protein, so only the protein slot is populated; the cdna and + genomic slots stay null (the c/g fan-out is ambiguous — no canonical pick is fabricated).""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=1.0, hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele( + session, "prot-digest", level="protein", clingen_allele_id="PA9", hgvs_p="NP_000537.3:p.Ala406Thr" + ) + _link( + session, + record, + measured, + is_authoritative=True, + group=_group(session, record, hgvs_p="NP_000537.3:p.Ala406Thr"), + ) + # A candidate fan-out member (cdna) in its own candidate group must NOT be pulled into the canonical triple. + candidate = _allele(session, "cand-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + candidate_group = _group(session, record, hgvs_c="NM_000546.6:c.1216G>A") + _link(session, record, candidate, is_authoritative=False, group=candidate_group) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "protein" + assert record_out.assay_level_digest == "prot-digest" + assert record_out.clingen_allele_id == "PA9" + # Only the protein slot; cdna/genomic stay null despite the candidate fan-out being linked. + assert record_out.mapped == MappedTriple( + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") + ) + + +@pytest.mark.integration +def test_unwritten_measured_group_serves_an_empty_triple(session, setup_lib_db_with_score_set): + """Between mapping and the projection step the authoritative link has no group: the triple is empty, + even with a protein allele linked to the record. Still one row per variant.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # projection_group defaults to None + _link(session, record, protein, is_authoritative=False) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "cdna" + assert record_out.mapped == MappedTriple() + + +@pytest.mark.integration +def test_genomic_assay_serves_a_coding_projection_with_no_allele(session, setup_lib_db_with_score_set): + """An intronic genomic change has a coding coordinate but no cdna VRS allele: the cdna slot comes off the + measured group's string alone, and the protein slot stays null (no consequence).""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + measured_group = _group(session, record, hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, measured, is_authoritative=True, group=measured_group) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.mapped == MappedTriple( + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000465.4:c.123+5A>G"), + ) + + +@pytest.mark.integration +def test_hgvs_string_rides_even_when_unparseable(session, setup_lib_db_with_score_set): + """The canonical string is always present; a splice/multivariant expression simply carries no block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="NM_000546.6:c.[197A>G;472T>C]", hgvs_splice="c.122-6T>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="c.76_78del") + measured = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True, group=_group(session, record, hgvs_c="c.76_78del")) + + [record_out] = get_lean_score_set_variants(session, score_set) + + # Multivariant, intronic, and indel expressions are all string-only (no position/ref/alt block). + assert record_out.hgvs_nt == HgvsField(hgvs="NM_000546.6:c.[197A>G;472T>C]") + assert record_out.hgvs_nt.position is None + assert record_out.hgvs_splice == HgvsField(hgvs="c.122-6T>A") + # The measured cdna slot carries the (unparseable) indel string alone. + assert record_out.mapped.cdna == HgvsField(hgvs="c.76_78del") + + +@pytest.mark.integration +def test_unmapped_variant_has_null_mapped_fields(session, setup_lib_db_with_score_set): + """A variant with no live mapping record keeps its submitted HGVS + score but null mapped fields.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.5, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == 0.5 + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.consequence is None + assert record_out.clingen_allele_id is None + assert record_out.assay_level_digest is None + assert record_out.assay_level is None + assert record_out.mapped == MappedTriple() # empty triple — no slots populated + + +@pytest.mark.integration +def test_no_vep_consequence_is_none(session, setup_lib_db_with_score_set): + """A mapped variant with no live VEP consequence has consequence None but keeps its digest + mapped HGVS.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna") + _link( + session, record, measured, is_authoritative=True, group=_group(session, record, hgvs_c="NM_000546.6:c.1216G>A") + ) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.consequence is None + assert record_out.assay_level_digest == "cdna-digest" + assert record_out.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + + +@pytest.mark.integration +def test_ordered_by_variant_number(session, setup_lib_db_with_score_set): + """Records come back ordered by the integer after '#', not insertion or lexical order.""" + score_set = setup_lib_db_with_score_set + for suffix in (10, 2, 1): + _variant(session, score_set, suffix, score=float(suffix)) + + records = get_lean_score_set_variants(session, score_set) + + assert [r.variant_urn for r in records] == [f"{score_set.urn}#{n}" for n in (1, 2, 10)] + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_assay_level(session, setup_lib_db_with_score_set): + """A re-map retires the old record and inserts a new one; as_of reconstructs the mapped layer live at + a past instant, while the immutable submitted HGVS and score are unaffected.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + + old_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", valid_from=T0) + old_allele = _allele(session, "old-digest", level="cdna") + _link( + session, + old_record, + old_allele, + is_authoritative=True, + group=_group(session, old_record, hgvs_c="NM_000546.6:c.1216G>A", valid_from=T0), + valid_from=T0, + ) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + + new_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.500A>T", valid_from=T1) + new_allele = _allele(session, "new-digest", level="cdna") + _link( + session, + new_record, + new_allele, + is_authoritative=True, + group=_group(session, new_record, hgvs_c="NM_000546.6:c.500A>T", valid_from=T1), + valid_from=T1, + ) + session.commit() + + [current] = get_lean_score_set_variants(session, score_set) + assert current.assay_level_digest == "new-digest" + assert current.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") + + [historical] = get_lean_score_set_variants(session, score_set, as_of=T1 - timedelta(days=1)) + assert historical.assay_level_digest == "old-digest" + assert historical.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + # Submitted HGVS and score are immutable — unaffected by as_of. + assert historical.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert historical.score == -2.3 + + # Before anything was mapped, the variant still appears with null mapped fields. + [pre_mapping] = get_lean_score_set_variants(session, score_set, as_of=T0 - timedelta(days=1)) + assert pre_mapping.assay_level_digest is None + assert pre_mapping.assay_level is None + assert pre_mapping.mapped == MappedTriple() + + +@pytest.mark.integration +@pytest.mark.parametrize("as_of", [None, T2], ids=["current", "as_of"]) +def test_per_variant_lookups_cannot_become_whole_table_reads(session, setup_lib_db_with_score_set, as_of): + """With nested loops and sequential scans disabled, every read of the per-variant tables must still be + keyed on the outer row: the query must not be able to turn into a hash join over the whole table.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", valid_from=T0) + measured = _allele(session, "cdna-digest", level="cdna") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + valid_from=T0, + ) + _link(session, record, measured, is_authoritative=True, group=measured_group, valid_from=T0) + _link(session, record, genomic, is_authoritative=False, group=measured_group, valid_from=T0) + _link(session, record, protein, is_authoritative=False, group=measured_group, valid_from=T0) + + with captured_statements(session) as captured: + get_lean_score_set_variants(session, score_set, as_of=as_of) + [(statement, parameters)] = [(s, p) for s, p in captured if "FROM variants" in s] + + assert ( + unkeyed_scans( + session, + statement, + parameters, + {"mapping_records": "variant_id", "mapping_record_alleles": "mapping_record_id"}, + ) + == [] + ) diff --git a/tests/lib/test_seqrepo.py b/tests/lib/test_seqrepo.py index 822d1a131..e37d1750e 100644 --- a/tests/lib/test_seqrepo.py +++ b/tests/lib/test_seqrepo.py @@ -7,7 +7,7 @@ from mavedb.lib.seqrepo import get_sequence_ids, _generate_nsa_options, seqrepo_versions, sequence_generator from mavedb.lib import seqrepo as seqrepo_lib -from tests.helpers.constants import TEST_SEQREPO_INITIAL_STATE +from tests.helpers.constants import TEST_SEQREPO_INITIAL_STATE, VALID_MD5_DIGEST @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) @@ -35,6 +35,18 @@ def test_get_sequence_ids_no_match(seqrepo): assert [] == result +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_ids_rejects_wildcards(entry, seqrepo): + alias = list(entry.keys())[0] + namespace = entry[alias]["namespace"] + assert get_sequence_ids(seqrepo, f"{namespace}:{entry[alias]['alias'][:4]}%") == [] + assert get_sequence_ids(seqrepo, f"{namespace}:%") == [] + + +def test_get_sequence_ids_rejects_digest_prefixes(seqrepo): + assert get_sequence_ids(seqrepo, VALID_MD5_DIGEST[:8]) == [] + + @pytest.mark.parametrize( "query,expected", [ @@ -44,8 +56,14 @@ def test_get_sequence_ids_no_match(seqrepo): # Namespace inferred ("ENST00000530893.6", [("ensembl", "ENST00000530893.6")]), ("NM_000551.3", [("refseq", "NM_000551.3")]), - # Hex digest (MD5/VMC) - ("01234abcde", [("MD5", "01234abcde%"), ("VMC", "GS_ASNKvN4=%")]), + # Complete hex digests only: MD5 (32 characters) and TRUNC512, as VMC (48) + ("0123456789ABCDEF0123456789abcdef", [("MD5", "0123456789abcdef0123456789abcdef")]), + ("0123456789abcdef" * 3, [("VMC", "GS_ASNFZ4mrze8BI0VniavN7wEjRWeJq83v")]), + ("01234abcde", [(None, "01234abcde")]), + # SeqRepo treats "%" as a LIKE wildcard, so any query containing one matches nothing. + ("ga4gh:SQ.%", []), + ("NM_%", []), + ("%", []), # No match, fallback to (None, query) ("notfound", [(None, "notfound")]), ], diff --git a/tests/lib/test_seqrepo_contract.py b/tests/lib/test_seqrepo_contract.py new file mode 100644 index 000000000..58e7383ab --- /dev/null +++ b/tests/lib/test_seqrepo_contract.py @@ -0,0 +1,143 @@ +"""Golden tests for the accession -> refget contract shared with dcd_mapping. + +A VRS allele digest covers the ``refgetAccession`` of the sequence its location sits on, and the allele +table deduplicates on that digest. The mapper and reverse translation each build alleles, so if they +resolve an accession to different sequences they mint different digests for the same variant and the +copies never merge. That happened: the mapper's fetcher chain lacked hgvs's SeqRepo-backed ``SeqFetcher``, +so cdot returned a transcript assembled from the genome (7207 nt) rather than NCBI's NM_007294.3 record +(7224 nt), and the mapper minted ``SQ.bh0R…`` where reverse translation minted ``SQ.jj1R…``. Nothing failed, +because both digests were internally consistent. + +The two repos run different VRS versions and cannot import each other, so the contract is pinned by a +fixture. ``tests/fixtures/canonical_accession_sequences.json`` is a byte-identical copy of the one in +dcd_mapping; it holds NCBI's records and the refget each must have. If either repo drifts, its copy of +these tests fails. Expected refgets are ``sha512t24u(sequence)``, so they are correct by construction. +""" + +# ruff: noqa: E402 +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +pytest.importorskip("biocommons.seqrepo") + +from biocommons.seqrepo import SeqRepo +from ga4gh.core import sha512t24u +from hgvs.dataproviders.seqfetcher import SeqFetcher +from hgvs.exceptions import HGVSDataNotAvailableError + +from mavedb.data_providers import services +from mavedb.lib.seqrepo import AmbiguousSequenceError, SequenceNotFoundError, resolve_refget +from mavedb.lib.vrs_utils import AlleleRefgetMismatchError, verify_allele_refget + +FIXTURE = Path(__file__).parents[1] / "fixtures" / "canonical_accession_sequences.json" +ACCESSIONS = json.loads(FIXTURE.read_text())["accessions"] + + +@pytest.fixture +def seqrepo_dir(tmp_path: Path) -> Path: + """Build a small SeqRepo of only the canonical fixture sequences, stored under ``refseq:``.""" + root = tmp_path / "seqrepo" + root.mkdir() + sr = SeqRepo(str(root), writeable=True) + for accession, record in ACCESSIONS.items(): + sr.store(record["sequence"], [{"namespace": "refseq", "alias": accession}]) + sr.commit() + return root + + +@pytest.fixture +def sr(seqrepo_dir: Path) -> SeqRepo: + return SeqRepo(str(seqrepo_dir)) + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_fixture_refget_is_digest_of_sequence(accession): + """Guards the fixture itself against a hand-edit that desynchronises sequence and refget.""" + record = ACCESSIONS[accession] + assert record["expected_refget"] == f"SQ.{sha512t24u(record['sequence'].encode())}" + assert record["expected_refget"] not in record["non_canonical_refgets"] + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_resolve_refget_is_the_canonical_refget(accession, sr): + record = ACCESSIONS[accession] + refget = resolve_refget(sr, accession) + assert refget == record["expected_refget"] + assert refget not in record["non_canonical_refgets"] + + +def test_missing_accession_is_reported(sr): + with pytest.raises(SequenceNotFoundError): + resolve_refget(sr, "NM_000000.1") + + +def test_more_than_one_sequence_is_an_error_not_a_pick(): + sr = MagicMock() + sr.aliases.find_aliases.return_value = [{"seq_id": "aaa"}, {"seq_id": "bbb"}] + with pytest.raises(AmbiguousSequenceError): + resolve_refget(sr, "NM_007294.3") + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_seqfetcher_serves_ncbi_records_ahead_of_the_fasta_fallback(accession, seqrepo_dir, monkeypatch): + """The chain feeding hgvs validation must resolve accessions from SeqRepo before the genome-assembled + FASTA fallback. The FASTA fetcher is stubbed to refuse, so a chain without the SeqRepo-backed fetcher in + front (the original defect in dcd_mapping) cannot serve the sequence. + """ + monkeypatch.setenv("HGVS_SEQREPO_DIR", str(seqrepo_dir)) + fasta = MagicMock(source="stub fasta") + fasta.fetch_seq.side_effect = HGVSDataNotAvailableError("no fasta in test") + with ( + patch.object(services, "FastaSeqFetcher", return_value=fasta), + patch.object(services, "GENOMIC_FASTA_FILES", ["stub.fna"]), + ): + chain = services.seqfetcher() + + assert isinstance(chain.seq_fetchers[0], SeqFetcher) + sequence = chain.fetch_seq(accession) + record = ACCESSIONS[accession] + assert sequence == record["sequence"] + assert f"SQ.{sha512t24u(sequence.encode())}" == record["expected_refget"] + + +def _allele(refget: str) -> dict: + return { + "type": "Allele", + "location": {"type": "SequenceLocation", "sequenceReference": {"refgetAccession": refget}}, + } + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_insert_check_accepts_the_canonical_refget(accession, sr): + refget = ACCESSIONS[accession]["expected_refget"] + verify_allele_refget(_allele(refget), f"{accession}:c.1A>G", sr, subject="test") + + +@pytest.mark.parametrize("accession", ACCESSIONS) +def test_insert_check_rejects_a_non_canonical_refget(accession, sr): + """The original failure: an allele built on a sequence that is not SeqRepo's for its accession.""" + bad = ACCESSIONS[accession]["non_canonical_refgets"][0] + with pytest.raises(AlleleRefgetMismatchError): + verify_allele_refget(_allele(bad), f"{accession}:c.1A>G", sr, subject="test") + + +def test_insert_check_inspects_every_member_of_a_block(sr): + good = ACCESSIONS["NM_007294.3"]["expected_refget"] + bad = ACCESSIONS["NM_007294.3"]["non_canonical_refgets"][0] + block = {"type": "CisPhasedBlock", "members": [_allele(good), _allele(bad)]} + with pytest.raises(AlleleRefgetMismatchError): + verify_allele_refget(block, "NM_007294.3:c.[1A>G;5C>T]", sr, subject="test") + + +def test_insert_check_ignores_accessions_seqrepo_is_not_the_authority_for(sr): + verify_allele_refget(_allele("SQ.anything"), "target_label:c.1A>G", sr, subject="test") + verify_allele_refget(_allele("SQ.anything"), "c.1A>G", sr, subject="test") + + +def test_insert_check_alerts_but_does_not_raise_for_an_unresolvable_accession(sr, caplog): + with caplog.at_level("ERROR"): + verify_allele_refget(_allele("SQ.anything"), "NM_000000.1:c.1A>G", sr, subject="test") + assert "Cannot verify the refget" in caplog.text diff --git a/tests/lib/test_term_systems.py b/tests/lib/test_term_systems.py new file mode 100644 index 000000000..962ce324e --- /dev/null +++ b/tests/lib/test_term_systems.py @@ -0,0 +1,14 @@ +"""Tests for the shared (system, prefix) -> Coding helper.""" + +import pytest + +from mavedb.lib.term_systems import coding + + +@pytest.mark.unit +def test_coding_builds_id_from_prefix_and_code(): + result = coding(("https://example.org/system", "ex"), "some_code") + + assert result.id == "ex:some_code" + assert result.code.root == "some_code" + assert result.system == "https://example.org/system" diff --git a/tests/lib/test_urn_redirects.py b/tests/lib/test_urn_redirects.py new file mode 100644 index 000000000..4c7410aae --- /dev/null +++ b/tests/lib/test_urn_redirects.py @@ -0,0 +1,162 @@ +"""Tests for forwarding URNs that publication has retired.""" + +# ruff: noqa: E402 + +import pytest + +# The module under test reaches fastapi and starlette directly, and arq, biocommons and cdot through +# mavedb.deps, all of which live in the `server` extra. Guarding the import keeps the core-dependency +# CI job skipping this module rather than failing to collect it. The DB fixtures need psycopg2. +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") +pytest.importorskip("arq") +pytest.importorskip("cdot") + +from mavedb.lib.temp_urns import generate_temp_urn +from mavedb.lib.urn_redirects import forwarded_path, record_urn_redirect +from mavedb.models.urn_redirect import UrnRedirect + +from tests.helpers.constants import VALID_EXPERIMENT_SET_URN, VALID_EXPERIMENT_URN, VALID_SCORE_SET_URN + +RETIRED_URN = "tmp:00000000-0000-4000-8000-000000000001" +LIVE_TMP_URN = "tmp:00000000-0000-4000-8000-000000000002" +PUBLISHED_URN = VALID_SCORE_SET_URN + +# Every entity type publication renames. Adding an entry extends the fixture and each parametrized +# test below, which are all driven by these keys. +# +# urn the URN setup_lib_db_with_score_set gives this entity +# route the collection its URN sits under in a request path +# score_set_path the attribute path reaching it from that score set, empty for the score set +# itself. Every type here is renamed by score set publication; an entity with +# its own publishing workflow would need the fixture to build it another way. +# sub_resource a path segment the API serves after its URN, or None where it serves none +ENTITY_TYPES = { + "score_set": { + "urn": VALID_SCORE_SET_URN, + "route": "score-sets", + "score_set_path": "", + "sub_resource": "scores", + }, + "experiment": { + "urn": VALID_EXPERIMENT_URN, + "route": "experiments", + "score_set_path": "experiment", + "sub_resource": "score-sets", + }, + "experiment_set": { + "urn": VALID_EXPERIMENT_SET_URN, + "route": "experiment-sets", + "score_set_path": "experiment.experiment_set", + "sub_resource": None, + }, +} + +ENTITY_TYPES_WITH_SUB_RESOURCE = [name for name, entity in ENTITY_TYPES.items() if entity["sub_resource"]] + + +@pytest.mark.integration +class TestRecordUrnRedirect: + def test_records_a_rename(self, session): + record_urn_redirect(session, RETIRED_URN, PUBLISHED_URN) + session.commit() + + redirect = session.query(UrnRedirect).one() + assert redirect.old_urn == RETIRED_URN + assert redirect.new_urn == PUBLISHED_URN + + def test_ignores_a_record_that_had_no_urn(self, session): + record_urn_redirect(session, None, PUBLISHED_URN) + session.commit() + + assert session.query(UrnRedirect).count() == 0 + + def test_ignores_a_rename_that_changes_nothing(self, session): + record_urn_redirect(session, PUBLISHED_URN, PUBLISHED_URN) + session.commit() + + assert session.query(UrnRedirect).count() == 0 + + +@pytest.mark.integration +class TestForwardedPath: + @pytest.fixture + def retired(self, session, setup_lib_db_with_score_set): + """Retire a temporary URN onto a public record of every entity type, keyed by type. + + A real record is needed, not just a row in the table: forwarding withholds a target it cannot + confirm is public, so a redirect pointing at nothing forwards nowhere. + """ + retired = {} + for entity_type, entity in ENTITY_TYPES.items(): + record = setup_lib_db_with_score_set + for attribute in filter(None, entity["score_set_path"].split(".")): + record = getattr(record, attribute) + + record.private = False + + # A redirect row is unique on the URN it retires, so every type needs its own. + retired_urn = generate_temp_urn() + record_urn_redirect(session, retired_urn, record.urn) + retired[entity_type] = (retired_urn, record) + + session.commit() + return retired + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_forwards_a_retired_urn(self, session, retired, entity_type): + retired_urn, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] + + assert forwarded_path(session, f"/api/v1/{route}/{retired_urn}") == f"/api/v1/{route}/{record.urn}" + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES_WITH_SUB_RESOURCE) + def test_forwards_a_sub_resource_of_a_retired_urn(self, session, retired, entity_type): + retired_urn, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] + sub_resource = ENTITY_TYPES[entity_type]["sub_resource"] + + assert ( + forwarded_path(session, f"/api/v1/{route}/{retired_urn}/{sub_resource}") + == f"/api/v1/{route}/{record.urn}/{sub_resource}" + ) + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_withholds_a_target_that_is_private(self, session, retired, entity_type): + retired_urn, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] + + record.private = True + session.commit() + + assert forwarded_path(session, f"/api/v1/{route}/{retired_urn}") is None + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_withholds_a_target_that_does_not_exist(self, session, entity_type): + """A deleted record leaves its redirect row behind. Without the fixture, none of these exist.""" + entity = ENTITY_TYPES[entity_type] + + record_urn_redirect(session, RETIRED_URN, entity["urn"]) + session.commit() + + assert forwarded_path(session, f"/api/v1/{entity['route']}/{RETIRED_URN}") is None + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_leaves_a_path_naming_no_temporary_urn_alone(self, session, retired, entity_type): + _, record = retired[entity_type] + route = ENTITY_TYPES[entity_type]["route"] + + assert forwarded_path(session, f"/api/v1/{route}/{record.urn}") is None + + def test_forwards_a_variant_of_a_retired_score_set(self, session, retired): + """A variant URN is built from its score set's, so the score set's redirect carries it.""" + retired_urn, score_set = retired["score_set"] + + assert forwarded_path(session, f"/api/v1/variants/{retired_urn}#4") == f"/api/v1/variants/{score_set.urn}#4" + + @pytest.mark.parametrize("entity_type", ENTITY_TYPES) + def test_leaves_a_live_temporary_urn_alone(self, session, retired, entity_type): + """An unpublished record still answers to its temporary URN, and has no row in the table.""" + route = ENTITY_TYPES[entity_type]["route"] + + assert forwarded_path(session, f"/api/v1/{route}/{LIVE_TMP_URN}") is None diff --git a/tests/lib/test_urns.py b/tests/lib/test_urns.py index 380bf0fc9..8bb56b8d0 100644 --- a/tests/lib/test_urns.py +++ b/tests/lib/test_urns.py @@ -1,9 +1,10 @@ -import pytest - -from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key +import importlib.util -pytestmark = pytest.mark.unit +import pytest +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker +from mavedb.lib.urns import lock_urn_assignment, score_set_urn_sort_key, variant_urn_sort_key # --------------------------------------------------------------------------- # Each of these pairs is one a lexical sort gets wrong. They are the reason the keys exist, so they are @@ -12,6 +13,7 @@ # --------------------------------------------------------------------------- +@pytest.mark.unit class TestScoreSetUrnSortKey: def test_unpadded_score_set_number_orders_numerically(self): urns = ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] @@ -66,6 +68,7 @@ def test_undecomposable_urns_still_order_deterministically_among_themselves(self assert sorted(urns, key=score_set_urn_sort_key) == ["tmp:a", "tmp:b", "tmp:c"] +@pytest.mark.unit class TestVariantUrnSortKey: def test_unpadded_variant_number_orders_numerically(self): urns = [f"urn:mavedb:00000001-a-1#{n}" for n in (2, 10, 1)] @@ -97,3 +100,29 @@ def test_urns_without_a_variant_suffix_sort_last_without_raising(self, urn): numbered = "urn:mavedb:00000001-a-1#1" assert sorted([urn, numbered], key=variant_urn_sort_key) == [numbered, urn] + + +@pytest.mark.integration +@pytest.mark.skipif(importlib.util.find_spec("psycopg2") is None, reason="needs the Postgres session fixture") +class TestLockUrnAssignment: + @staticmethod + def _other_session_can_take_lock(session) -> bool: + other = sessionmaker(bind=session.get_bind())() + try: + return other.execute(text("SELECT pg_try_advisory_xact_lock(hashtext('mavedb.urn_assignment'))")).scalar() + finally: + other.rollback() + other.close() + + def test_lock_excludes_other_sessions_until_commit(self, session): + lock_urn_assignment(session) + assert self._other_session_can_take_lock(session) is False + + session.commit() + assert self._other_session_can_take_lock(session) is True + + def test_lock_is_released_on_rollback(self, session): + lock_urn_assignment(session) + session.rollback() + + assert self._other_session_can_take_lock(session) is True diff --git a/tests/lib/test_variant_annotations_script.py b/tests/lib/test_variant_annotations_script.py new file mode 100644 index 000000000..1a654bc3e --- /dev/null +++ b/tests/lib/test_variant_annotations_script.py @@ -0,0 +1,95 @@ +# ruff: noqa: E402 +"""Tests for the variant_annotations CLI's current_annotation_summary resolution. + +The score set is the entry point, but status is resolved per-allele through the live mapping links — +so a shared allele's status counts even when a *different* score set's run produced it. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.scripts.variant_annotations import current_annotation_summary +from tests.helpers.constants import TEST_MINIMAL_VARIANT + + +def _variant_mapped_to_allele(session, score_set, allele): + """A variant in the score set with a live mapping record + live authoritative link to ``allele``.""" + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + session.add(variant) + session.commit() + + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="genomic", mapping_api_version="test.0.0" + ) + session.add(record) + session.commit() + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=allele.id, is_authoritative=True + ) + ) + session.commit() + return variant + + +def test_summary_counts_allele_status_from_another_score_sets_run(session, setup_lib_db_with_score_set, job_run): + """An allele-subject status produced by a *different* score set's run (or none) still counts for a + score set whose variant currently maps to that shared allele.""" + score_set = setup_lib_db_with_score_set + allele = Allele( + vrs_digest="summary-shared", level="genomic", clingen_allele_id="CA1", post_mapped={"type": "Allele"} + ) + session.add(allele) + session.commit() + _variant_mapped_to_allele(session, score_set, allele) + + # Event written with no owning score set (e.g. a different score set's run touched the shared allele). + session.add( + AnnotationEvent( + annotation_type=AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="created", + job_run_id=job_run.id, + score_set_id=None, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.CLINGEN_ALLELE_ID.value, Disposition.PRESENT.value)) == 1 + + +def test_summary_includes_variant_subject_status(session, setup_lib_db_with_score_set, job_run): + """Variant-subject types (mapping/RT/LDH) are counted per variant, off the score set's variants.""" + score_set = setup_lib_db_with_score_set + allele = Allele(vrs_digest="summary-vs", level="genomic", post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + variant = _variant_mapped_to_allele(session, score_set, allele) + + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.VRS_MAPPING.value, Disposition.PRESENT.value)) == 1 diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py new file mode 100644 index 000000000..efb8f1d33 --- /dev/null +++ b/tests/lib/test_variant_detail.py @@ -0,0 +1,649 @@ +# ruff: noqa: E402 +"""Integration tests for the variant-detail envelope assembly (``lib/variant_detail.py``). + +These pin the composition: the flat assay fields (level, target/reference HGVS, digest, ClinGen id) +off the live record + authoritative allele; the spec-pure Cat-VRS + MaveDB mode/member-relations off +the on-the-fly builder; the digest-keyed annotation map; the per-calibration classifications +(primary first, filterable by visibility); scores/counts passthrough; and the ``is_current`` / +``superseded_by_score_set`` version standing. Cat-VRS/annotation internals are covered in their own suites — here +we assert they are wired in and keyed correctly. +""" + +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.context import load_annotation_context_inputs, variant_annotation_context +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.lib.allele_identity import AlleleDerivation +from mavedb.lib.score_set_variants import HgvsField, MappedTriple +from mavedb.lib.variant_detail import get_variant_detail, load_variant_detail_inputs +from mavedb.lib.mondo import get_generic_disease_term +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.user import User +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT, TEST_USER +from tests.helpers.util.query_plan import captured_statements + +# A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this, so it must parse. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_g=None, hgvs_c=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped=_post_mapped(), + clingen_allele_id=clingen_allele_id, + hgvs_g=hgvs_g, + hgvs_c=hgvs_c, + hgvs_p=hgvs_p, + ) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _group(session, record, *, hgvs_c=None, hgvs_g=None, hgvs_p=None): + group = ProjectionGroup( + mapping_record_id=record.id, score_set_id=record.score_set_id, hgvs_c=hgvs_c, hgvs_g=hgvs_g, hgvs_p=hgvs_p + ) + session.add(group) + session.commit() + return group + + +def _link(session, record, allele, *, is_authoritative=False, group=None): + link = MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=group.id if group is not None else None, + ) + session.add(link) + session.commit() + return link + + +def _vep(session, allele, value): + session.add( + VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + ) + session.commit() + + +def _calibration(session, score_set, *, primary, classifications): + """A calibration with functional classifications; each ``classifications`` entry is + ``(functional, [variants...])`` and its m2m membership is set to those variants.""" + user = session.query(User).filter(User.username == TEST_USER["username"]).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + title=f"cal-{'primary' if primary else 'secondary'}", + primary=primary, + private=False, + disease_term=get_generic_disease_term(session), + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + for functional, variants in classifications: + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, label=functional, functional_classification=functional + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +@pytest.mark.integration +def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): + """Mode 1 (coding measured): flat assay fields off the record + authoritative allele, spec-pure + Cat-VRS in projection mode, and a digest-keyed annotation map covering the linked alleles.""" + score_set = setup_lib_db_with_score_set + variant = _variant( + session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr" + ) + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + # A synonymous cousin: an nt encoder of the same protein consequence in a *different* projection group + # — a distinct, unmeasured variant carried by the reverse-translation fan. + cousin = _allele(session, "cousin-digest", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") + # The measured group holds the measured cdna, its genomic projection, and the protein consequence; the + # cousin is in its own candidate group. + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + _link(session, record, protein, group=measured_group) + _link(session, record, cousin, group=_group(session, record, hgvs_c="NM_000546.6:c.1218C>T")) + _vep(session, measured, "missense_variant") + + detail = get_variant_detail(session, variant) + + assert detail.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + assert detail.urn == variant.urn + assert detail.scores == {"score": -2.3} + assert detail.assay_level == "cdna" + assert detail.target_hgvs == "c.1216G>A" # submitted, target frame + assert detail.reference_hgvs == "NM_000546.6:c.1216G>A" # mapped, reference frame + assert detail.assay_level_digest == "cdna-digest" + assert detail.clingen_allele_id == "CA123" + assert detail.mode == "projection" + assert detail.molecular_representation is not None + assert detail.molecular_representation["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar — every linked allele is a + # member, and vice versa (the cousin included). + assert len(detail.molecular_representation["members"]) == len(detail.alleles) + # The alleles sidecar carries one identity per linked allele, keyed by digest: level + + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation + + # derivation + projection_of. + assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} + measured_identity = detail.alleles["cdna-digest"] + assert measured_identity.level == "cdna" + assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c + assert measured_identity.clingen_allele_id == "CA123" + assert measured_identity.relation is None # the defining allele has no relation to itself + # The measured allele is the focus; derivation describes the *other* members relative to it, so the + # focus carries none. It pairs with its genomic projection sibling. + assert measured_identity.is_focus is True + assert measured_identity.derivation is None + assert measured_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.is_focus is False + # Nucleotide assay: the genomic sibling is a precise projection, paired back to the measured cdna. + assert genomic_identity.level == "genomic" + assert genomic_identity.derivation == "projection" + assert genomic_identity.projection_of == "cdna-digest" + protein_identity = detail.alleles["prot-digest"] + assert protein_identity.level == "protein" + assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p + # The protein member is a translation of the measured coding allele (member -> defining)... + assert protein_identity.relation == "translation_of" + # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent). It is not + # a c<->g counterpart, so it pairs with nothing. + assert protein_identity.derivation == "projection" + assert protein_identity.projection_of is None + # The synonymous cousin (different projection group) is a member wearing co_encodes (structural), and + # labelled `convergent` (provenance) — a distinct, precise change that converges on the measured + # consequence, not an ambiguous candidate and not a projection of the measured change. It pairs with + # nothing (its group has a single member). + cousin_identity = detail.alleles["cousin-digest"] + assert cousin_identity.level == "cdna" + assert cousin_identity.relation == "co_encodes" + assert cousin_identity.derivation == "convergent" + assert cousin_identity.projection_of is None + # Annotations keyed by digest, covering all linked alleles; VEP rode in on the measured allele. + assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} + assert detail.annotations["cdna-digest"].vep is not None + assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" + assert detail.is_current is True + assert detail.superseded_by_score_set is None + + +@pytest.mark.integration +def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_score_set): + """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, the nt + members `encode` the defining protein allele, and the fanned-out c/g candidates are labelled + `candidate` (ambiguous) while still pairing to each other via projection_of. The protein apex is in + no pair (group None). The `relation` (structural) and `derivation` (provenance) axes are independent: + the coding candidate is `encodes` + `candidate` here, versus `translation_of` + `projection` for the + protein member of a nucleotide assay.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele(session, "prot-digest", level="protein", clingen_allele_id="PA9") + coding = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, measured, is_authoritative=True) # protein apex — group None + # One reverse-translation projection pair: a coding candidate paired with its genomic projection. + candidate_group = _group(session, record, hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, coding, group=candidate_group) + _link(session, record, genomic, group=candidate_group) + + detail = get_variant_detail(session, variant) + + assert detail.assay_level == "protein" + assert detail.target_hgvs == "p.Ala406Thr" # protein assay -> submitted protein string + assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" + assert detail.assay_level_digest == "prot-digest" + assert detail.mode == "reverse_translation" + # Defining protein allele is the focus (no relation/derivation to itself), and is in no c/g pair. + prot = detail.alleles["prot-digest"] + assert prot.level == "protein" + assert prot.relation is None + assert prot.is_focus is True + assert prot.derivation is None + assert prot.projection_of is None + # The coding member `encodes` the protein (structural relation) but is an ambiguous `candidate` + # (provenance) — the two axes disagree by design — and pairs to its genomic projection. + coding_identity = detail.alleles["cdna-digest"] + assert coding_identity.relation == "encodes" + assert coding_identity.derivation == "candidate" + assert coding_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.relation == "encodes" + assert genomic_identity.derivation == "candidate" + assert genomic_identity.projection_of == "cdna-digest" + + +@pytest.mark.integration +def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, setup_lib_db_with_score_set): + """Links written before reverse translation ran carry a NULL projection_group: focus/derivation are + still computed (is_focus from is_authoritative; projection from the assay level), but projection_of + degrades to null — nothing pairs until the data is re-processed.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # no projection_group (pre-RT) + _link(session, record, protein) + + detail = get_variant_detail(session, variant) + + assert detail.alleles["cdna-digest"].is_focus is True + assert detail.alleles["cdna-digest"].derivation is None + assert detail.alleles["cdna-digest"].projection_of is None # no group -> no sibling + assert detail.alleles["prot-digest"].derivation == "projection" + assert detail.alleles["prot-digest"].projection_of is None + + +@pytest.mark.integration +def test_mapped_triple_carries_a_coding_projection_with_no_allele(session, setup_lib_db_with_score_set): + """An intronic genomic change has a coding coordinate but no cdna VRS allele: ``mapped`` serves the + measured group's hgvs_c anyway, and no protein consequence.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + measured_group = _group( + session, record, hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p=None + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + + detail = get_variant_detail(session, variant) + + assert detail.mapped == MappedTriple( + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000465.4:c.123+5A>G"), + ) + assert set(detail.alleles) == {"gen-digest"} + assert detail.alleles["gen-digest"].projection_of is None + + +@pytest.mark.integration +def test_protein_link_in_the_measured_group_pairs_with_nothing(session, setup_lib_db_with_score_set): + """The protein consequence shares the measured group but is what the pair encodes, not another spelling of + it, so it pairs with nothing; the nucleotide pair still points at each other.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + measured_group = _group( + session, + record, + hgvs_c="NM_000546.6:c.1216G>A", + hgvs_g="NC_000017.11:g.7676154C>T", + hgvs_p="NP_000537.3:p.Ala406Thr", + ) + # Protein linked first, so a pairing that ignored level would pick it as the cdna allele's partner. + _link(session, record, protein, group=measured_group) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + + variant_alleles = get_variant_detail(session, variant).alleles + assert variant_alleles["prot-digest"].projection_of is None + assert variant_alleles["cdna-digest"].projection_of == "gen-digest" + assert variant_alleles["gen-digest"].projection_of == "cdna-digest" + + nucleotide_focus = get_allele_detail(session, measured, focus_digests={"cdna-digest"}).alleles + assert nucleotide_focus["cdna-digest"].projection_of == "gen-digest" + assert nucleotide_focus["gen-digest"].projection_of == "cdna-digest" + + protein_focus = get_allele_detail(session, protein, focus_digests={"prot-digest"}).alleles + assert protein_focus["prot-digest"].projection_of is None + assert protein_focus["cdna-digest"].projection_of is None + + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _gnomad_link(session, allele, gnomad_id, *, valid_from=None): + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier=gnomad_id, + db_version="v4.1", + allele_count=1, + allele_number=100, + allele_frequency=0.01, + ) + session.add(gnomad_variant) + session.flush() + link = GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id) + if valid_from is not None: + link.valid_from = valid_from + session.add(link) + session.flush() + return link + + +@pytest.mark.integration +def test_categorical_variant_cross_references_linked_gnomad_and_clinvar_records(session, setup_lib_db_with_score_set): + """The live gnomAD and ClinVar links reach the Cat-VRS as identifier-only ``mappings``: the measured + change's records are an exactMatch, a convergent encoding's a relatedMatch.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + genomic = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA123") + cousin = _allele(session, "cousin-digest", level="cdna") + measured_group = _group(session, record) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + _link(session, record, cousin, group=_group(session, record)) + + _gnomad_link(session, genomic, "17-7676154-C-T") + _gnomad_link(session, cousin, "17-7676152-G-A") + control = ClinvarControl( + db_identifier="12345", + gene_symbol="TP53", + clinical_significance="Pathogenic", + clinical_review_status="reviewed by expert panel", + db_name="ClinVar", + db_version="11_2024", + clinvar_variation_id="376654", + ) + session.add(control) + session.flush() + session.add(ClinvarAlleleLink(allele_id=measured.id, clinvar_control_id=control.id)) + session.commit() + + detail = get_variant_detail(session, variant) + + assert detail.molecular_representation is not None + mappings = { + (m["coding"]["system"], m["coding"]["code"]): m["relation"] for m in detail.molecular_representation["mappings"] + } + assert mappings == { + ("https://reg.clinicalgenome.org/", "CA123"): "exactMatch", + ("https://www.ncbi.nlm.nih.gov/clinvar/variation/", "376654"): "exactMatch", + ("https://gnomad.broadinstitute.org", "17-7676154-C-T"): "exactMatch", + ("https://gnomad.broadinstitute.org", "17-7676152-G-A"): "relatedMatch", + } + + +@pytest.mark.integration +def test_categorical_variant_mappings_follow_as_of(session, setup_lib_db_with_score_set): + """``as_of`` reaches the cross-references: a gnomAD match replaced at T1 maps to the old ID at T0.""" + variant = _variant(session, setup_lib_db_with_score_set, 1, data={"score_data": {"score": -2.3}}) + record = _record(session, variant, assay_level="cdna") + measured = _allele(session, "cdna-digest", level="cdna") + genomic = _allele(session, "gen-digest", level="genomic") + measured_group = _group(session, record) + for row in ( + _link(session, record, measured, is_authoritative=True, group=measured_group), + _link(session, record, genomic, group=measured_group), + measured_group, + ): + row.valid_from = T0 + record.valid_from = T0 + _gnomad_link(session, genomic, "17-1-A-G", valid_from=T0).retire(at=T1) + _gnomad_link(session, genomic, "17-1-A-T", valid_from=T1) + session.commit() + + def gnomad_ids(as_of): + detail = get_variant_detail(session, variant, as_of=as_of) + assert detail.molecular_representation is not None + return [ + m["coding"]["code"] + for m in detail.molecular_representation["mappings"] + if m["coding"]["system"] == "https://gnomad.broadinstitute.org" + ] + + assert gnomad_ids(datetime(2020, 6, 1, tzinfo=timezone.utc)) == ["17-1-A-G"] + assert gnomad_ids(None) == ["17-1-A-T"] + + +@pytest.mark.integration +def test_unmapped_variant_has_null_molecular_layer(session, setup_lib_db_with_score_set): + """No live record: flat mapped fields and Cat-VRS are null, the annotation map is empty, and the + variant reads as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.5}}, hgvs_nt="c.1A>T") + + detail = get_variant_detail(session, variant) + + assert detail.scores == {"score": 0.5} + assert detail.assay_level is None + assert detail.target_hgvs == "c.1A>T" # submitted still surfaces (no assay level -> nucleotide) + assert detail.reference_hgvs is None + assert detail.assay_level_digest is None + assert detail.clingen_allele_id is None + assert detail.molecular_representation is None + assert detail.mode is None + assert detail.alleles == {} + assert detail.annotations == {} + assert detail.is_current is True + + +@pytest.mark.integration +def test_classifications_are_per_calibration_primary_first(session, setup_lib_db_with_score_set): + """A variant carries one classification per calibration that classifies it; the primary sorts first.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + + detail = get_variant_detail(session, variant) + + assert [(c.primary, c.classification.functional_classification.value) for c in detail.classifications] == [ + (True, "abnormal"), + (False, "normal"), + ] + + +@pytest.mark.integration +def test_visible_calibration_ids_filters_classifications(session, setup_lib_db_with_score_set): + """Only classifications from calibrations the caller resolved as visible are returned.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + visible = _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + + detail = get_variant_detail(session, variant, visible_calibration_ids={visible.id}) + + assert [c.calibration_id for c in detail.classifications] == [visible.id] + + +@pytest.mark.integration +def test_superseded_variant_self_describes(session, setup_lib_db_with_score_set): + """When a superseding version is passed in, the variant reports it rather than reading as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + newer = SimpleNamespace(urn="urn:mavedb:00000001-a-2") # only .urn is read + + detail = get_variant_detail(session, variant, superseding_score_set=newer) + + assert detail.is_current is False + assert detail.superseded_by_score_set == "urn:mavedb:00000001-a-2" + + +def _seed_mapped_variants(session, score_set, count): + """``count`` coding-measured variants, each with a genomic projection, a protein apex shared by all of + them, VEP on the measured allele, and membership in one primary calibration's classification.""" + protein = _allele(session, "shared-prot", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + variants = [] + for index in range(count): + variant = _variant(session, score_set, index + 1, data={"score_data": {"score": float(index)}}) + record = _record(session, variant, hgvs_assay_level=f"NM_000546.6:c.{1216 + index}G>A") + measured = _allele(session, f"cdna-{index}", level="cdna", hgvs_c=f"NM_000546.6:c.{1216 + index}G>A") + genomic = _allele(session, f"gen-{index}", level="genomic", hgvs_g=f"NC_000017.11:g.{7676154 + index}C>T") + measured_group = _group( + session, + record, + hgvs_c=measured.hgvs_c, + hgvs_g=genomic.hgvs_g, + hgvs_p=protein.hgvs_p, + ) + _link(session, record, measured, is_authoritative=True, group=measured_group) + _link(session, record, genomic, group=measured_group) + _link(session, record, protein, group=measured_group) + _vep(session, measured, "missense_variant") + variants.append(variant) + _calibration(session, score_set, primary=True, classifications=[("abnormal", variants)]) + return variants + + +@pytest.mark.integration +def test_batched_details_equal_single_variant_details(session, setup_lib_db_with_score_set): + """A chunk's envelopes are identical to building each variant on its own, unmapped variants included.""" + score_set = setup_lib_db_with_score_set + variants = [*_seed_mapped_variants(session, score_set, 3), _variant(session, score_set, 99)] + + inputs = load_variant_detail_inputs(session, variants) + + for variant in variants: + assert inputs.detail_for(variant) == get_variant_detail(session, variant) + + +@pytest.mark.integration +def test_batched_contexts_equal_single_variant_contexts(session, setup_lib_db_with_score_set): + """A chunk's annotation contexts are identical to building each variant on its own.""" + score_set = setup_lib_db_with_score_set + variants = [*_seed_mapped_variants(session, score_set, 3), _variant(session, score_set, 99)] + + inputs = load_annotation_context_inputs(session, variants) + + for variant in variants: + assert inputs.context_for(variant) == variant_annotation_context(session, variant) + + +@pytest.mark.integration +@pytest.mark.parametrize( + "load", [load_variant_detail_inputs, load_annotation_context_inputs], ids=["detail", "annotation_context"] +) +def test_batched_loads_run_a_fixed_number_of_queries(session, setup_lib_db_with_score_set, load): + """Loading a chunk costs the same number of queries however many variants it holds.""" + score_set = setup_lib_db_with_score_set + variants = _seed_mapped_variants(session, score_set, 6) + # Seeding commits, which expires the variants; reload them so only the loader's own queries are counted. + for variant in variants: + session.refresh(variant) + + with captured_statements(session) as one: + load(session, variants[:1]) + with captured_statements(session) as six: + load(session, variants) + + assert len(six) == len(one) + + +@pytest.mark.integration +@pytest.mark.parametrize("folded", [True, False], ids=["folded", "fold_in_missed"]) +def test_variant_and_allele_detail_label_members_alike(session, setup_lib_db_with_score_set, folded): + """Both views label each member relative to the measured allele through one function, so they agree, + including when the measured allele has no projection group and an ungrouped member sits beside it.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + record = _record(session, variant) + measured = _allele(session, "measured", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + twin = _allele(session, "twin", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + ungrouped = _allele(session, "ungrouped", level="genomic", hgvs_g="NC_000017.11:g.7676160C>T") + convergent = _allele(session, "convergent", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") + protein = _allele(session, "protein", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + twin_group = _group(session, record, hgvs_c=measured.hgvs_c, hgvs_g=twin.hgvs_g) + _link(session, record, measured, is_authoritative=True, group=twin_group if folded else None) + _link(session, record, twin, group=twin_group) + _link(session, record, ungrouped) + _link(session, record, convergent, group=_group(session, record, hgvs_c=convergent.hgvs_c)) + _link(session, record, protein) + + variant_alleles = get_variant_detail(session, variant).alleles + allele_alleles = get_allele_detail(session, measured, focus_digests={"measured"}).alleles + + labels = {digest: (identity.relation, identity.derivation) for digest, identity in variant_alleles.items()} + assert labels == {digest: (identity.relation, identity.derivation) for digest, identity in allele_alleles.items()} + assert labels["ungrouped"] == ("co_encodes", AlleleDerivation.CONVERGENT) + assert labels["twin"][0] == ("coordinate_representation_of" if folded else "co_encodes") diff --git a/tests/lib/test_variant_translations.py b/tests/lib/test_variant_translations.py index 832152c7f..addb9a0c8 100644 --- a/tests/lib/test_variant_translations.py +++ b/tests/lib/test_variant_translations.py @@ -91,3 +91,75 @@ def test_fully_overlapping_second_call_inserts_nothing(self, session): session.commit() rows = session.scalars(select(VariantTranslation)).all() assert len(rows) == 2 + + +def _draft(digest, *, hgvs_c=None, hgvs_p=None, refget=None, level="cdna"): + from mavedb.models.allele import Allele + + post_mapped = {"type": "Allele", "location": {"sequenceReference": {"refgetAccession": refget}}} if refget else None + return Allele(vrs_digest=digest, level=level, hgvs_c=hgvs_c, hgvs_p=hgvs_p, post_mapped=post_mapped) + + +@pytest.mark.integration +class TestGetOrCreateAllele: + """One allele per digest, and one allele per HGVS expression.""" + + def test_creates_and_flushes_a_new_allele(self, session): + from mavedb.lib.variant_translations import get_or_create_allele + + allele = get_or_create_allele(session, _draft("digest-a", hgvs_c="NM_000001.1:c.1A>G")) + + assert allele.id is not None + + def test_returns_the_existing_allele_for_the_same_digest(self, session): + from mavedb.lib.variant_translations import get_or_create_allele + + first = get_or_create_allele(session, _draft("digest-a", hgvs_c="NM_000001.1:c.1A>G")) + again = get_or_create_allele(session, _draft("digest-a", hgvs_c="NM_000001.1:c.1A>G")) + + assert again is first + + def test_alleles_without_an_hgvs_expression_do_not_conflict(self, session): + from mavedb.lib.variant_translations import get_or_create_allele + + get_or_create_allele(session, _draft("digest-a")) + get_or_create_allele(session, _draft("digest-b")) + + def test_same_hgvs_under_a_different_digest_is_a_conflict_that_explains_itself(self, session): + from mavedb.lib.variant_translations import AlleleIdentityConflictError, get_or_create_allele + + hgvs = "NM_007294.3:c.5448A>T" + get_or_create_allele(session, _draft("digest-good", hgvs_c=hgvs, refget="SQ.jj1R")) + + with pytest.raises(AlleleIdentityConflictError) as excinfo: + get_or_create_allele(session, _draft("digest-bad", hgvs_c=hgvs, refget="SQ.bh0R")) + + message = str(excinfo.value) + for expected in (hgvs, "digest-good", "digest-bad", "SQ.jj1R", "SQ.bh0R", "AlleleIdentityConflictError"): + assert expected in message + + def test_a_conflict_does_not_add_the_draft_or_poison_the_session(self, session): + from mavedb.lib.variant_translations import AlleleIdentityConflictError, get_or_create_allele + from mavedb.models.allele import Allele + + hgvs = "NM_000001.1:c.1A>G" + get_or_create_allele(session, _draft("digest-good", hgvs_c=hgvs)) + with pytest.raises(AlleleIdentityConflictError): + get_or_create_allele(session, _draft("digest-bad", hgvs_c=hgvs)) + + digests = set(session.scalars(select(Allele.vrs_digest))) + assert digests == {"digest-good"} + get_or_create_allele(session, _draft("digest-other", hgvs_c="NM_000001.1:c.2A>G")) + + def test_the_database_index_backstops_a_writer_that_skips_the_check(self, session): + """A concurrent writer past the application check must still be stopped, by name.""" + from sqlalchemy.exc import IntegrityError + + hgvs = "NM_000001.1:c.1A>G" + session.add(_draft("digest-a", hgvs_c=hgvs)) + session.flush() + session.add(_draft("digest-b", hgvs_c=hgvs)) + + with pytest.raises(IntegrityError, match="uq_alleles_hgvs"): + session.flush() + session.rollback() diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index f01650a05..1ef1e1a58 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -6,6 +6,7 @@ hgvs_from_vrs_allele, is_hgvs_g, is_hgvs_p, + score_from_variant_data, ) from tests.helpers.constants import ( TEST_GA4GH_DIGEST, @@ -17,6 +18,28 @@ TEST_VALID_POST_MAPPED_VRS_HAPLOTYPE, ) +### Tests for score_from_variant_data function ### + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + ({"score_data": {"score": -2.3}}, -2.3), + ({"score_data": {"score": 0}}, 0.0), # a real 0.0 score is not "missing" + ({"score_data": {"score": "1.5"}}, 1.5), # numeric strings coerce + ({"score_data": {"score": None}}, None), # explicit NA + ({"score_data": {}}, None), # no score key + ({"count_data": {"c": 1}}, None), # no score_data block + ({"score_data": {"score": True}}, None), # a JSON bool is not a score + ({"score_data": {"score": "NA"}}, None), # non-numeric string + ({}, None), + (None, None), + ], +) +def test_score_from_variant_data(data, expected): + assert score_from_variant_data(data) == expected + + ### Tests for hgvs_from_vrs_allele function ### @@ -48,6 +71,18 @@ def test_get_hgvs_from_post_mapped_cis_phased_block(): assert result is None +def test_get_hgvs_from_post_mapped_cis_phased_block_combine_cis(): + # combine_cis collapses the cis-phased members into one bracketed expression. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, combine_cis=True) + assert result == "NM_003345:p.[Asp5Phe;Asp5Phe]" + + +def test_get_hgvs_from_post_mapped_single_allele_combine_cis_is_unbracketed(): + # A single-variant post-mapped allele is unaffected by combine_cis. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, combine_cis=True) + assert result == TEST_HGVS_IDENTIFIER + + def test_get_hgvs_from_post_mapped_single_allele_vrs_1(): with pytest.raises(ValueError): get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X) @@ -73,6 +108,26 @@ def test_get_hgvs_from_post_mapped_invalid_structure(): get_hgvs_from_post_mapped({"invalid_key": "InvalidType"}) +def test_hgvs_from_vrs_allele_null_or_empty_expressions(): + # A VRS allele may carry `expressions: null` or `[]` — that is "no HGVS", not a crash. + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": None}) is None + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": []}) is None + + +def test_get_hgvs_from_post_mapped_member_without_expression(): + # Regression: a cis-phased block member whose `expressions` is null must yield None, not raise + # `TypeError: 'NoneType' object is not subscriptable` (which previously killed the CAR job). + block = { + "type": "CisPhasedBlock", + "members": [ + {"type": "Allele", "expressions": [{"value": "NM_003345:p.Asp5Phe"}]}, + {"type": "Allele", "expressions": None}, + ], + } + assert get_hgvs_from_post_mapped(block) is None + assert get_hgvs_from_post_mapped(block, combine_cis=True) is None + + ### Tests for get_id_from_post_mapped function ### diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index 2e9e1faef..98b51a0d3 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -1,242 +1,340 @@ -"""Unit tests for mavedb.lib.vep — covers response parsing for both Variant Recoder and VEP APIs. +"""Unit tests for mavedb.lib.vep — the Ensembl release lookup, the resolution-delegation adapter, and +the allele linker. -These tests mock the underlying HTTP call (request_with_backoff) and assert that the parsing -logic correctly handles the actual Ensembl REST API response shapes. +The resolution flow itself — VEP querying, transcript matching, the Recoder fallback, how forms combine +— lives in ``variant_annotation.lib.vep`` and is covered by that library's own tests. This module tests +only what the api owns: that ``resolve_consequences`` delegates to the library over an Ensembl client +and re-keys the result by input HGVS, the release lookup, and the DB linker (run against the DB). """ +# ruff: noqa: E402 + +from datetime import date, timedelta from unittest.mock import MagicMock, patch import pytest -from mavedb.lib.vep import get_functional_consequence, run_variant_recoder +pytest.importorskip("variant_annotation") +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + +from sqlalchemy import select + +from variant_annotation.lib.vep import ( + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ConsequenceSource, + VepInput, +) + +from mavedb.lib.vep import ( + VepLinkVerdict, + get_ensembl_release, + link_vep_consequences_to_alleles, + resolve_consequences, +) +from mavedb.models.allele import Allele +from mavedb.models.enums.vep import VepConsequenceSource +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +def _resolved( + hgvs, + term, + *, + source=ConsequenceSource.TRANSCRIPT, + terms=None, + matched=None, +) -> ConsequenceResolution: + """A RESOLVED resolution the linker can persist, defaulting to a transcript-matched single term.""" + return ConsequenceResolution( + input=VepInput(hgvs=hgvs), + outcome=ConsequenceOutcome.RESOLVED, + consequence_terms=list(terms) if terms is not None else [term], + most_severe_consequence=term, + source=source, + matched_transcript=matched, + ) + + +def _absent(hgvs) -> ConsequenceResolution: + """A resolution for an input VEP answered with no consequence — a genuine empty.""" + return ConsequenceResolution(input=VepInput(hgvs=hgvs), outcome=ConsequenceOutcome.ABSENT) + + +### Tests for get_ensembl_release function ### -def _mock_response(data) -> MagicMock: - """Return a mock requests.Response whose .json() returns *data*.""" - mock = MagicMock() - mock.json.return_value = data - return mock +@pytest.mark.asyncio +@pytest.mark.unit +async def test_get_ensembl_release_returns_the_clients_release(): + """The release lookup returns whatever the Ensembl client's software_release reports (it version-keys + the skip). The /info/software call itself is the client's, covered by the library.""" + with patch("mavedb.lib.vep.EnsemblRestClient.software_release", return_value="116"): + assert await get_ensembl_release() == "116" -# --------------------------------------------------------------------------- -# Realistic Variant Recoder response payloads -# -# Ensembl Variant Recoder returns a list of single-key dicts. The key is the -# called allele (e.g. "T"). The allele dict contains input, hgvsg, hgvsc, etc. -# --------------------------------------------------------------------------- - -RECODER_RESPONSE_SINGLE = [ - { - "T": { - "hgvsc": [ - "NM_007294.4:c.5G>A", - ], - "spdi": ["NC_000017.11:43094691:C:T"], - "input": "NP_009225.1:p.Cys2Tyr", - "hgvsp": ["NP_009225.1:p.Cys2Tyr"], - "hgvsg": [ - "NC_000017.11:g.43094692C>T", - "LRG_292:g.100C>T", # non-NC_ entry — should be filtered out - ], - } - } -] - -RECODER_RESPONSE_MULTIPLE = [ - { - "A": { - "input": "NM_007294.4:c.5G>T", - "hgvsg": ["NC_000017.11:g.43094692C>A"], - } - }, - { - "T": { - "input": "NM_007294.4:c.5G>A", - "hgvsg": ["NC_000017.11:g.43094692C>T"], - } - }, -] - -RECODER_RESPONSE_MULTI_ALLELE = [ - { - # Both allele keys share the same "input" value; each has distinct NC_ hgvsg entries. - "CAT": { - "input": "NP_009225.1:p.Val1696His", - "hgvsg": [ - "NC_000017.11:g.43063938_43063940delinsATG", - "LRG_292:g.154044_154046delinsCAT", - ], - "hgvsc": ["NM_007294.4:c.5086_5088delinsCAT"], - }, - "CAC": { - "input": "NP_009225.1:p.Val1696His", - "hgvsg": [ - "NC_000017.11:g.43063938_43063940inv", - "LRG_292:g.154044_154046inv", - ], - "hgvsc": ["NM_007294.4:c.5086_5088inv"], - }, - } -] - -RECODER_RESPONSE_NO_HGVSG = [ - { - "T": { - "input": "NP_009225.1:p.Xxx1Yyy", - "hgvsc": ["NM_007294.4:c.5G>A"], - # deliberately no "hgvsg" key - } - } -] - - -# --------------------------------------------------------------------------- -# Realistic VEP /vep/human/hgvs response payloads -# -# VEP returns a list of dicts where "input" and "most_severe_consequence" are -# *top-level* keys in each element (unlike Recoder where "input" is nested). -# --------------------------------------------------------------------------- - -VEP_RESPONSE_SINGLE = [ - { - "input": "NM_007294.4:c.5G>A", - "most_severe_consequence": "missense_variant", - "id": "rs80357382", - "seq_region_name": "17", - } -] - -VEP_RESPONSE_MULTIPLE = [ - { - "input": "NM_007294.4:c.5G>A", - "most_severe_consequence": "missense_variant", - }, - { - "input": "NM_007294.4:c.10C>T", - "most_severe_consequence": "synonymous_variant", - }, -] - - -# --------------------------------------------------------------------------- -# run_variant_recoder -# --------------------------------------------------------------------------- +### Tests for resolve_consequences (the delegation adapter) ### @pytest.mark.asyncio @pytest.mark.unit -class TestRunVariantRecoder: - async def test_extracts_nc_genomic_hgvs_from_real_response_format(self): - """Correctly extracts NC_ genomic HGVS from the nested allele-key response structure.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_SINGLE)): - result = await run_variant_recoder(["NP_009225.1:p.Cys2Tyr"]) - - assert result == {"NP_009225.1:p.Cys2Tyr": ["NC_000017.11:g.43094692C>T"]} - - async def test_filters_non_nc_genomic_hgvs(self): - """Non-NC_ entries in hgvsg (e.g. LRG accessions) are excluded from the result.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_SINGLE)): - result = await run_variant_recoder(["NP_009225.1:p.Cys2Tyr"]) - - returned_hgvsg = result.get("NP_009225.1:p.Cys2Tyr", []) - assert all(h.startswith("NC_") for h in returned_hgvsg) - assert not any("LRG" in h for h in returned_hgvsg) - - async def test_handles_multiple_inputs(self): - """Multiple input variants in one batch are all mapped correctly.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_MULTIPLE)): - result = await run_variant_recoder(["NM_007294.4:c.5G>T", "NM_007294.4:c.5G>A"]) - - assert result == { - "NM_007294.4:c.5G>T": ["NC_000017.11:g.43094692C>A"], - "NM_007294.4:c.5G>A": ["NC_000017.11:g.43094692C>T"], - } - - async def test_handles_multiple_allele_keys_per_element(self): - """A single response element with multiple allele keys (e.g. CAT, CAC) collects NC_ hgvsg from all of them. - - Variant Recoder can return multiple possible allele representations for the same input variant - in a single element dict. Each allele key is independent and may carry different hgvsg entries. - """ - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_MULTI_ALLELE)): - result = await run_variant_recoder(["NP_009225.1:p.Val1696His"]) - - genomic_hgvsg = result.get("NP_009225.1:p.Val1696His", []) - assert "NC_000017.11:g.43063938_43063940delinsATG" in genomic_hgvsg - assert "NC_000017.11:g.43063938_43063940inv" in genomic_hgvsg - assert all(h.startswith("NC_") for h in genomic_hgvsg) - - async def test_returns_empty_for_variant_without_hgvsg(self): - """A variant whose allele dict has no hgvsg field is not included in the result.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_NO_HGVSG)): - result = await run_variant_recoder(["NP_009225.1:p.Xxx1Yyy"]) +class TestResolveConsequences: + """resolve_consequences is a thin adapter over the shared library orchestration: it runs the + library's resolve_consequences over an Ensembl client and re-keys the result by input HGVS. The + resolution flow itself is the library's, covered by its own tests.""" + + async def test_delegates_and_keys_results_by_hgvs(self): + """The library's list-in-input-order result comes back re-keyed by each input's HGVS.""" + a = VepInput(hgvs="NM_1:c.5G>A") + b = VepInput(hgvs="NM_1:c.10C>T") + library_result = [_resolved(a.hgvs, "missense_variant"), _absent(b.hgvs)] + with patch("mavedb.lib.vep.resolve_consequences_kernel", MagicMock(return_value=library_result)) as kernel: + result = await resolve_consequences([a, b]) + + kernel.assert_called_once() # delegated to the library, not re-implemented here + assert set(result) == {a.hgvs, b.hgvs} + assert result[a.hgvs].outcome is ConsequenceOutcome.RESOLVED + assert result[b.hgvs].outcome is ConsequenceOutcome.ABSENT + + async def test_empty_input_short_circuits_without_calling_the_library(self): + """No inputs -> no client built and no library call, just an empty result.""" + with patch("mavedb.lib.vep.resolve_consequences_kernel", MagicMock()) as kernel: + result = await resolve_consequences([]) assert result == {} + kernel.assert_not_called() - async def test_skips_non_dict_allele_values(self): - """If an allele value is not a dict (e.g. null in the response), it is skipped gracefully.""" - response_with_null_allele = [{"T": None}] - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(response_with_null_allele)): - result = await run_variant_recoder(["some_hgvs"]) - - assert result == {} - async def test_input_field_at_allele_level_not_top_level(self): - """Regression: the 'input' field is nested inside the allele key, not at the top level of each list element. +### Tests for link_vep_consequences_to_alleles function ### - The old buggy implementation called entry.get("input") on the outer dict, - which always returned None and silently dropped all results. - """ - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(RECODER_RESPONSE_SINGLE)): - result = await run_variant_recoder(["NP_009225.1:p.Cys2Tyr"]) - # If "input" were read from the wrong level, result would be empty. - assert result != {} - assert "NP_009225.1:p.Cys2Tyr" in result +def _make_allele(session, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele.""" + allele = Allele(vrs_digest=vrs_digest, level=level) + session.add(allele) + session.commit() + session.refresh(allele) + return allele -# --------------------------------------------------------------------------- -# get_functional_consequence -# --------------------------------------------------------------------------- +def _live_rows_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() -@pytest.mark.asyncio -@pytest.mark.unit -class TestGetFunctionalConsequence: - async def test_extracts_most_severe_consequence(self): - """Parses input and most_severe_consequence from the top-level VEP response structure.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(VEP_RESPONSE_SINGLE)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A"]) +def _all_rows_for(session, allele_id): + return session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele_id)).all() - assert result == {"NM_007294.4:c.5G>A": "missense_variant"} - async def test_maps_none_when_consequence_absent(self): - """When most_severe_consequence is missing from an entry, the HGVS maps to None.""" - response = [{"input": "NM_007294.4:c.5G>A"}] - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(response)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A"]) +def test_link_vep_creates_new_consequence(session): + """A consequence for an allele with no live row creates a single live row carrying the headline term + and its full resolution provenance, and is reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") - assert result == {"NM_007294.4:c.5G>A": None} - - async def test_skips_entries_without_input_key(self): - """Entries that have no 'input' key are skipped entirely.""" - response = [{"most_severe_consequence": "missense_variant"}] - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(response)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A"]) - - assert result == {} - - async def test_handles_multiple_variants(self): - """All variants in a batch response are extracted correctly.""" - with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response(VEP_RESPONSE_MULTIPLE)): - result = await get_functional_consequence(["NM_007294.4:c.5G>A", "NM_007294.4:c.10C>T"]) - - assert result == { - "NM_007294.4:c.5G>A": "missense_variant", - "NM_007294.4:c.10C>T": "synonymous_variant", - } + verdicts = link_vep_consequences_to_alleles( + session, + { + allele.id: _resolved( + "NM_007294.4:c.5G>A", + "missense_variant", + terms=["missense_variant", "splice_region_variant"], + matched="NM_007294.4", + ) + }, + source_version="116", + access_date=date.today(), + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.CREATED} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + assert live[0].access_date == date.today() + # Full resolution provenance is persisted, not just the headline term (#772). + assert live[0].consequence_terms == ["missense_variant", "splice_region_variant"] + assert live[0].consequence_source == VepConsequenceSource.transcript + assert live[0].matched_transcript == "NM_007294.4" + # The resolution-rule version is stamped so the skip can tell a rule-fixed row from a stale one. + assert live[0].resolver_version == RESOLVER_VERSION + + +def test_link_vep_unchanged_bumps_version_and_fills_provenance_in_place(session): + """Re-confirming an unchanged headline term at a new release advances source_version/access_date in + place — no supersede — and fills in the resolution provenance a pre-#772 row was missing. The allele + is reported UNCHANGED (status preexisting) so the caller need not re-query consequence state.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, + { + allele.id: _resolved( + "NM_1:c.5G>A", "missense_variant", source=ConsequenceSource.TRANSCRIPT, matched="NM_1.4" + ) + }, + source_version="116", + access_date=date.today(), + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} + # One row, still live, never retired — version, date, and provenance advanced in place. + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 1 + assert all_rows[0].valid_to is None + assert all_rows[0].source_version == "116" + assert all_rows[0].access_date == date.today() + assert all_rows[0].consequence_source == VepConsequenceSource.transcript + assert all_rows[0].matched_transcript == "NM_1.4" + # A pre-column row (NULL resolver_version) is filled in place, not superseded. + assert all_rows[0].resolver_version == RESOLVER_VERSION + + +def test_link_vep_source_change_same_term_advances_in_place(session): + """A better provenance for the *same* headline term (most_severe -> transcript) is not a value + change: it advances in place rather than churning a new history row. History records term changes.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + consequence_source=VepConsequenceSource.most_severe, + source_version="116", + access_date=date.today() - timedelta(days=1), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, + { + allele.id: _resolved( + "NM_1:c.5G>A", "missense_variant", source=ConsequenceSource.TRANSCRIPT, matched="NM_1.4" + ) + }, + source_version="116", + access_date=date.today(), + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} + rows = _all_rows_for(session, allele.id) + assert len(rows) == 1 # no supersede + assert rows[0].consequence_source == VepConsequenceSource.transcript + assert rows[0].matched_transcript == "NM_1.4" + + +def test_link_vep_changed_consequence_supersedes(session): + """A changed headline term retires the live row and inserts the successor — exactly one live row, + keyed on allele_id, with the old one preserved as retired history.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, + {allele.id: _resolved("NM_1:c.5G>A", "missense_variant")}, + source_version="116", + access_date=date.today(), + ) + session.commit() + + assert verdicts == {allele.id: VepLinkVerdict.CREATED} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 2 + assert len([r for r in all_rows if r.valid_to is not None]) == 1 + + +def test_link_vep_absent_leaves_live_row_untouched(session): + """A transient absent result must not overwrite a held consequence: the live row is left intact + (value, version, and date). The held consequence is reported RETAINED_ON_ABSENCE (status + preexisting) — the allele still has a live consequence, it just was not re-confirmed this run.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + verdicts = link_vep_consequences_to_alleles( + session, {allele.id: _absent("NM_1:c.5G>A")}, source_version="116", access_date=date.today() + ) + session.commit() + + # RETAINED_ON_ABSENCE, not UNCHANGED: the prior value was held because VEP found nothing this run, + # a case the caller surfaces distinctly (job metadata + a loud warning). + assert verdicts == {allele.id: VepLinkVerdict.RETAINED_ON_ABSENCE} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + # Not re-confirmed -> neither version nor access_date advanced. + assert live[0].source_version == "115" + assert live[0].access_date == date.today() - timedelta(days=90) + + +def test_link_vep_absent_with_no_live_row_writes_nothing(session): + """An absent result for an allele with no live row writes nothing and leaves the allele out of the + verdict map (the caller reads that as a no-result and re-queries next run), mirroring gnomAD's + no-match handling.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + verdicts = link_vep_consequences_to_alleles( + session, {allele.id: _absent("NM_1:c.5G>A")}, source_version="116", access_date=date.today() + ) + session.commit() + + assert verdicts == {} + assert len(_all_rows_for(session, allele.id)) == 0 + + +def test_link_vep_errored_is_never_linked(session): + """An ERRORED resolution must never reach the value table (that would overwrite a held consequence + with a failure). It is ignored defensively and leaves no verdict — failures live in the event stream.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + verdicts = link_vep_consequences_to_alleles( + session, + { + allele.id: ConsequenceResolution( + input=VepInput(hgvs="NM_1:c.5G>A"), outcome=ConsequenceOutcome.ERRORED, error="boom" + ) + }, + source_version="116", + access_date=date.today(), + ) + session.commit() - async def test_raises_if_more_than_200_variants(self): - """Passing more than 200 HGVS strings raises ValueError before any HTTP call.""" - with pytest.raises(ValueError, match="maximum of 200"): - await get_functional_consequence(["NM_007294.4:c.1A>T"] * 201) + assert verdicts == {} + assert len(_all_rows_for(session, allele.id)) == 0 diff --git a/tests/lib/test_vrs.py b/tests/lib/test_vrs.py new file mode 100644 index 000000000..fae8dda67 --- /dev/null +++ b/tests/lib/test_vrs.py @@ -0,0 +1,96 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.vrs — deserialization of stored post_mapped dicts into GA4GH VRS objects.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from tests.helpers.constants import ( + TEST_SEQUENCE_LOCATION_ACCESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, +) + + +@pytest.mark.unit +class TestVariationExtractionUnit: + @pytest.mark.parametrize( + "variation_version", + [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], + ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], + ) + def test_vrs_object_from_post_mapped_variation(self, variation_version): + result = vrs_object_from_mapped_variant(variation_version).model_dump() + + assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION + assert result["location"]["start"] == 5 + assert result["location"]["end"] == 6 + + @pytest.mark.parametrize( + "allele_dict, expected_state_type, expected_state_fields", + [ + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE, + "LiteralSequenceExpression", + {"sequence": "F"}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, + "ReferenceLengthExpression", + {"length": 5, "repeatSubunitLength": 5}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + "LengthExpression", + {"length": 5}, + ), + ], + ids=["lse_state", "rle_state", "length_expression_state"], + ) + def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): + result = vrs_object_from_mapped_variant(allele_dict).model_dump() + + assert result["state"]["type"] == expected_state_type + for k, v in expected_state_fields.items(): + assert result["state"][k] == v + + def test_unknown_allele_state_type_raises_value_error(self): + allele_dict = { + **TEST_VALID_POST_MAPPED_VRS_ALLELE, + "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, + } + + with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): + vrs_object_from_mapped_variant(allele_dict) + + @pytest.mark.parametrize( + "member_dict, expected_state_type", + [ + (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), + ], + ids=["lse_members", "rle_members", "length_expression_members"], + ) + def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): + mapping_results = {"type": "CisPhasedBlock", "members": [deepcopy(member_dict), deepcopy(member_dict)]} + + result = vrs_object_from_mapped_variant(mapping_results).model_dump() + + assert result["type"] == "CisPhasedBlock" + assert len(result["members"]) == 2 + assert result["members"][0]["state"]["type"] == expected_state_type + + +@pytest.mark.integration +class TestVrsIntegration: + def test_vrs_object_from_persisted_post_mapped(self, setup_lib_db_with_mapped_variant): + variation = vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + + assert variation is not None + assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py new file mode 100644 index 000000000..931a00eb8 --- /dev/null +++ b/tests/lib/test_vrs_utils.py @@ -0,0 +1,408 @@ +# ruff: noqa: E402 + +import logging +from copy import deepcopy +from types import SimpleNamespace + +import pytest + +pytest.importorskip("ga4gh.vrs") +# mavedb.lib.vrs_utils imports ga4gh.vrs.extras.translator.AlleleTranslator, which needs hgvs (an +# optional "server" extra) even though ga4gh.vrs itself does not. +pytest.importorskip("hgvs") + +from ga4gh.core.models import iriReference +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + LiteralSequenceExpression, + Range, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, +) + +from mavedb.lib import vrs_utils +from mavedb.lib.vrs_utils import ( + _rle_to_lse, + canonical_variation_document, + identify_allele, + identify_variation, + normalize_and_identify, + translate_hgvs_to_variation, +) + +_SQ = "SQ." + "a" * 32 + +# A digest a reused ga4gh AlleleTranslator might leave on a component before it is +# re-identified — stale from the Merkle cache, or carried over from a sibling variant. +_STALE_ID = "ga4gh:VA." + "Z" * 32 + + +def _allele(start: int, alt: str) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + 1, + ), + state=LiteralSequenceExpression(sequence=alt), + ) + + +def _patch_component_translator(monkeypatch, alleles_by_hgvs: dict[str, Allele]) -> None: + """Stub the per-component single-allele translator that translate_hgvs_to_variation calls.""" + + def _fake(hgvs: str, translator=None) -> Allele: + return alleles_by_hgvs[hgvs] + + monkeypatch.setattr(vrs_utils, "translate_hgvs_to_vrs", _fake) + + +def _stub_normalize(monkeypatch) -> None: + """Make normalization a no-op so identification can be exercised without a seqrepo. + + Identification (the regression surface) runs for real; only the proxy-backed + normalize step is stubbed out. + """ + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: allele) + + +def _translator() -> SimpleNamespace: + return SimpleNamespace(data_proxy=object()) + + +def test_single_variant_returns_a_bare_allele(monkeypatch): + allele = _allele(1000, "G") + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) + + assert isinstance(result, Allele) + assert not isinstance(result, CisPhasedBlock) + + +def test_single_variant_is_reidentified_not_left_with_translator_digest(monkeypatch): + # ga4gh's translate_from (do_normalize=False) stamps id via plain ga4gh_identify on + # the reused translator; translate_hgvs_to_variation must re-identify so the persisted + # vrs_digest reflects current content rather than that carried-over value. + allele = _allele(1000, "G") + allele.id = _STALE_ID + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) + + assert result.id is not None + assert result.id != _STALE_ID + assert result.id.startswith("ga4gh:VA.") + + +def test_same_position_distinct_alts_get_distinct_digests(monkeypatch): + # Regression: NC_000002.12:g.214809459A>C and g.214809459A>T are different MaveDB + # variants that cannot encode the same codon, yet were merged onto one Allele row + # (and thus one coding equivalent like NM_000465.4:c.109_111delinsCGA). The single + # component returned by the reused translator carried a stale/shared digest, which + # the digest-keyed get_or_create_allele then deduplicated. Re-identification must + # give distinct content distinct digests so they never collide. + a_c = _allele(214809458, "C") + a_t = _allele(214809458, "T") + a_c.id = a_t.id = _STALE_ID # what the unfixed path would persist for both + _patch_component_translator( + monkeypatch, + { + "NC_000002.12:g.214809459A>C": a_c, + "NC_000002.12:g.214809459A>T": a_t, + }, + ) + _stub_normalize(monkeypatch) + translator = _translator() + + res_c = translate_hgvs_to_variation("NC_000002.12:g.214809459A>C", translator=translator) + res_t = translate_hgvs_to_variation("NC_000002.12:g.214809459A>T", translator=translator) + + assert res_c.id != _STALE_ID + assert res_t.id != _STALE_ID + assert res_c.id != res_t.id + + +def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) + + assert isinstance(result, CisPhasedBlock) + assert len(result.members) == 2 + assert result.id is not None and result.id.startswith("ga4gh:CPB.") + + +def test_cis_phased_block_digest_is_order_independent(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) + + forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) + reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=_translator()) + + # The same biological cis-phased set dedups to one row regardless of component ordering. + assert forward.id == reverse.id + + +def test_identify_variation_clears_stale_block_digest(): + block = CisPhasedBlock(members=[_allele(1000, "G"), _allele(1002, "C")]) + block.digest = "STALE" + + digest = identify_variation(block) + + assert digest != "STALE" + assert digest.startswith("ga4gh:CPB.") # recomputed from content, not the stale cache + + +# A location id left over from before normalization moved the span. The allele id comes out +# correct while the location it points at still claims the un-normalized interval. +_STALE_LOCATION_ID = "ga4gh:SL." + "Y" * 32 + + +def test_identify_allele_restamps_a_stale_location_id(): + """``ga4gh_identify`` writes only the id of the object it is handed, never a sub-object's.""" + reference = _allele(1000, "G") + identify_allele(reference) + assert isinstance(reference.location, SequenceLocation) + + drifted = _allele(1000, "G") + assert isinstance(drifted.location, SequenceLocation) + drifted.location.id = _STALE_LOCATION_ID + + identify_allele(drifted) + + assert drifted.location.id == reference.location.id + assert drifted.location.id != _STALE_LOCATION_ID + + +def test_an_identified_location_id_restates_its_own_digest(): + """The invariant that makes drift detectable by string comparison rather than recomputation.""" + allele = _allele(1000, "G") + identify_allele(allele) + assert isinstance(allele.location, SequenceLocation) + + assert allele.location.id == f"ga4gh:SL.{allele.location.digest}" + + +def test_identify_variation_restamps_a_stale_member_location_id(): + """A block embeds allele copies, so a drifted member location rides into the block unnoticed.""" + member = _allele(10, "A") + assert isinstance(member.location, SequenceLocation) + member.location.id = _STALE_LOCATION_ID + block = CisPhasedBlock(members=[member, _allele(20, "G")]) # type: ignore[call-arg] + + identify_variation(block) + + embedded = block.members[0] + assert isinstance(embedded, Allele) + assert isinstance(embedded.location, SequenceLocation) + assert embedded.location.id != _STALE_LOCATION_ID + assert embedded.location.id == f"ga4gh:SL.{embedded.location.digest}" + + +def _rle_allele(start: int, *, length: int, repeat_subunit_length: int) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + repeat_subunit_length, + ), + state=ReferenceLengthExpression(length=length, repeatSubunitLength=repeat_subunit_length), + ) + + +def test_rle_to_lse_tiles_repeat_subunit(): + # repeatSubunitLength=2 reads 2 bases from the proxy; length=4 tiles them out to "ACAC". + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = _rle_to_lse(rle, location, data_proxy) + + assert isinstance(result, LiteralSequenceExpression) + assert result.sequence.root == "ACAC" + + +def test_normalize_and_identify_coerces_rle_to_lse(monkeypatch): + # Regression: normalization can yield an RLE for an indel, but dcd_mapping stores LSE; the + # finalize step must coerce so the same variant doesn't hash to two digests (and duplicate). + rle = _rle_allele(10, length=2, repeat_subunit_length=2) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: rle) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = normalize_and_identify(rle, data_proxy=data_proxy) + + assert isinstance(result.state, LiteralSequenceExpression) + assert result.id is not None and result.id.startswith("ga4gh:VA.") + + +# The unions below (location.sequenceReference, location.start, rle.length) carry IRI-reference +# and Range variants that a fully-resolved indel allele never has. _rle_to_lse and the RLE branch +# of normalize_and_identify guard the inlined-and-integer contract with asserts; these tests pin +# that the guards fire rather than letting an IRI/Range slip into the digest computation as a +# silent AttributeError or wrong sequence read. +_NEVER_READ = SimpleNamespace(get_sequence=lambda identifier, start, end: pytest.fail("proxy read despite bad input")) + + +def test_rle_to_lse_rejects_iri_sequence_reference(): + location = SequenceLocation(sequenceReference=iriReference("seqref:unresolved"), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_start(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=Range([10, 12]), end=14) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_length(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=Range([2, 4]), repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_normalize_and_identify_rejects_iri_location_for_rle(monkeypatch): + # If normalization ever returned an RLE allele whose location is an unresolved IRI reference, + # _rle_to_lse could not read a sequence from it; the call-site assert must catch this before + # the digest is (mis)computed rather than crashing deeper with an AttributeError. + bad = Allele( + location=iriReference("loc:unresolved"), + state=ReferenceLengthExpression(length=2, repeatSubunitLength=2), + ) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: bad) + + with pytest.raises(AssertionError): + normalize_and_identify(bad, data_proxy=_NEVER_READ) + + +# --------------------------- canonical_variation_document --------------------------- +# +# The ingest guard: identifiers arriving from the mapper are assertions, not identity. These pin the +# behaviour that keeps `vrs_digest` — the dedup key the allele graph hangs off — true of its own content. + + +def _document(start: int, alt: str, *, identifier: str | None = None) -> dict: + """A serialized, correctly-identified allele; ``identifier`` overrides the id with a supplied one.""" + allele = _allele(start, alt) + allele.id = identify_allele(allele) + document = allele.model_dump(mode="json", exclude_none=True) + if identifier is not None: + document["id"] = identifier + return document + + +def test_canonicalization_is_a_no_op_for_a_correct_identifier(caplog): + """The overwhelming majority of incoming alleles are already right; they must pass through unchanged.""" + correct = identify_allele(_allele(10, "A")) + + document, identifier = canonical_variation_document(_document(10, "A", identifier=correct), subject="v1") + + assert identifier == correct + assert document["id"] == correct + assert "mismatch" not in caplog.text + + +def test_a_stale_identifier_is_replaced_and_warned_about(caplog): + """The drift case: an id minted over different content than the document now carries.""" + stale = identify_allele(_allele(99, "T")) + caplog.set_level(logging.WARNING) + + document, identifier = canonical_variation_document(_document(10, "A", identifier=stale), subject="urn:x#1") + + assert identifier != stale + assert identifier == identify_allele(_allele(10, "A")) + assert document["id"] == identifier + assert "urn:x#1" in caplog.text + assert stale in caplog.text + + +def test_an_absent_identifier_is_minted_without_a_warning(caplog): + caplog.set_level(logging.WARNING) + document = _document(10, "A") + document.pop("id", None) + + _, identifier = canonical_variation_document(document, subject="v1") + + assert identifier == identify_allele(_allele(10, "A")) + assert "mismatch" not in caplog.text + + +def test_the_same_content_canonicalizes_to_the_same_bytes(): + """Two writers, two serializations, one identifier is what corrupts a normalized export.""" + verbose = _document(10, "A") + verbose["digest"] = verbose["id"].split(".")[-1] + verbose["location"]["sequenceReference"]["label"] = "NC_000001.11" + verbose["extensions"] = [] + + terse = _document(10, "A") + terse.pop("digest", None) + + assert canonical_variation_document(verbose, subject="v")[0] == canonical_variation_document(terse, subject="v")[0] + + +def test_canonicalization_does_not_mutate_the_caller_s_document(): + document = _document(10, "A", identifier="ga4gh:VA.definitelyWrong") + before = deepcopy(document) + + canonical_variation_document(document, subject="v") + + assert document == before + + +def test_a_cis_phased_block_is_canonicalized_too(): + """CisPhasedBlock members embed allele copies, which is how one writer's convention leaks.""" + block = CisPhasedBlock(members=[_allele(10, "A"), _allele(20, "G")]) # type: ignore[call-arg] + block.id = "ga4gh:CPB.stale" + document = block.model_dump(mode="json", exclude_none=True) + + canonical, identifier = canonical_variation_document(document, subject="v") + + assert identifier.startswith("ga4gh:CPB.") + assert canonical["id"] == identifier + + +def test_canonicalization_repairs_a_stale_location_id(): + """The observed drift shape: a correct allele id over a location id minted pre-normalization. + + This is what put 938 identifiers on more than one coordinate span in + ``urn:mavedb:00000662-0-1`` — the allele level was repaired and the location level was not. + """ + document = _document(10, "A") + document["location"]["id"] = _STALE_LOCATION_ID + + canonical, _ = canonical_variation_document(document, subject="v") + + assert canonical["location"]["id"] != _STALE_LOCATION_ID + assert canonical["location"]["id"] == f"ga4gh:SL.{canonical['location']['digest']}" + + +def test_two_spans_that_shared_one_location_id_canonicalize_apart(): + """One key, two bodies is what corrupts a keyed export; distinct spans must get distinct ids.""" + first = _document(10, "A") + second = _document(11, "A") + second["location"]["id"] = first["location"]["id"] + + canonical_first, _ = canonical_variation_document(first, subject="v") + canonical_second, _ = canonical_variation_document(second, subject="v") + + assert canonical_first["location"]["id"] != canonical_second["location"]["id"] diff --git a/tests/lib/uniprot/test_utils.py b/tests/lib/uniprot/test_utils.py index 7afa278c2..d0643623b 100644 --- a/tests/lib/uniprot/test_utils.py +++ b/tests/lib/uniprot/test_utils.py @@ -1,12 +1,10 @@ import pytest -from mavedb.lib.validation.exceptions import ValidationError -from mavedb.lib.uniprot.utils import infer_db_name_from_sequence_accession - +from mavedb.lib.uniprot.utils import id_mapping_query_accession, infer_db_name_from_sequence_accession from tests.helpers.constants import ( + VALID_CHR_ACCESSION, VALID_NT_ACCESSION, VALID_PRO_ACCESSION, - VALID_CHR_ACCESSION, VALID_UNIPROT_ACCESSION, ) @@ -21,6 +19,20 @@ def test_infer_db_name_from_sequence_accession_np(): assert result == "RefSeq_Protein" +# Both versioned and unversioned ENSP and ENST accessions must route to Ensembl_Protein and Ensembl_Transcript, respectively; +# the versioned case guards that validate_ensembl_identifier's version handling continues to accept it. +@pytest.mark.parametrize("ensembl_protein_accession", ["ENSP00000418960", "ENSP00000418960.3"]) +def test_infer_db_name_from_sequence_accession_ensp(ensembl_protein_accession): + result = infer_db_name_from_sequence_accession(ensembl_protein_accession) + assert result == "Ensembl_Protein" + + +@pytest.mark.parametrize("ensembl_transcript_accession", ["ENST00000418960", "ENST00000418960.3"]) +def test_infer_db_name_from_sequence_accession_enst(ensembl_transcript_accession): + result = infer_db_name_from_sequence_accession(ensembl_transcript_accession) + assert result == "Ensembl_Transcript" + + @pytest.mark.parametrize("invalid_accession", ["XP_000000", VALID_CHR_ACCESSION]) def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): with pytest.raises(NotImplementedError): @@ -28,10 +40,30 @@ def test_infer_db_name_from_sequence_accession_invalid(invalid_accession): def test_infer_db_name_from_non_refseq_accession(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession(VALID_UNIPROT_ACCESSION) def test_infer_db_name_from_sequence_accession_empty_string(): - with pytest.raises(ValidationError): + with pytest.raises(NotImplementedError): infer_db_name_from_sequence_accession("") + + +def test_id_mapping_query_accession_strips_refseq_nucleotide_version(): + assert id_mapping_query_accession("NM_002878.4", "RefSeq_Nucleotide") == "NM_002878" + + +def test_id_mapping_query_accession_unversioned_refseq_nucleotide_unchanged(): + assert id_mapping_query_accession("NM_002878", "RefSeq_Nucleotide") == "NM_002878" + + +@pytest.mark.parametrize( + "accession, from_db", + [ + ("NP_002869.3", "RefSeq_Protein"), + ("ENST00000345365.11", "Ensembl_Transcript"), + ("ENSP00000338790.6", "Ensembl_Protein"), + ], +) +def test_id_mapping_query_accession_keeps_version_for_other_databases(accession, from_db): + assert id_mapping_query_accession(accession, from_db) == accession diff --git a/tests/lib/workflow/test_job_factory.py b/tests/lib/workflow/test_job_factory.py index 4ea1b5d0b..a00360697 100644 --- a/tests/lib/workflow/test_job_factory.py +++ b/tests/lib/workflow/test_job_factory.py @@ -26,6 +26,25 @@ def test_create_job_run_persists_preset_params_from_definition(self, job_factory assert job_run.job_params["param1"] == "value1" assert job_run.job_params["param2"] == "value2" + @pytest.mark.parametrize( + "pipeline_value, expected", + [({}, False), ({"force": True}, True)], + ids=["default", "set by pipeline"], + ) + def test_create_job_run_takes_optional_params_from_the_pipeline_or_their_default( + self, job_factory, sample_job_definition, pipeline_value, expected + ): + job_def = {**sample_job_definition, "optional_params": {"force": False}} + job_run = job_factory.create_job_run( + job_def=job_def, + correlation_id="test-correlation-id", + pipeline_params={"required_param": "required_value", "param1": "override", **pipeline_value}, + pipeline_id=1, + ) + + assert job_run.job_params["force"] is expected + assert job_run.job_params["param1"] == "value1" + def test_create_job_run_raises_error_for_missing_params(self, job_factory, sample_job_definition): incomplete_params = {"param1": "new_value1"} # Missing param2 diff --git a/tests/lib/workflow/test_kickoff.py b/tests/lib/workflow/test_kickoff.py new file mode 100644 index 000000000..f81879b12 --- /dev/null +++ b/tests/lib/workflow/test_kickoff.py @@ -0,0 +1,260 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from unittest.mock import AsyncMock, Mock, patch + +from arq.jobs import Job as ArqJob +from sqlalchemy import select + +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.user import User +from mavedb.worker.lib.managers.utils import FAST_LANE_QUEUE_NAME +from tests.helpers.constants import TEST_USER + + +def _mock_entrypoint(*, id=42, job_function="start_pipeline", urn="urn:mavedb:job-fixed", retry_count=0): + return Mock(id=id, job_function=job_function, urn=urn, retry_count=retry_count) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetUnit: + """Unit tests with PipelineFactory and ARQ mocked out.""" + + @pytest.fixture(autouse=True) + def setup(self): + self.db = Mock() + self.redis = AsyncMock() + self.score_set = Mock(id=99, urn="urn:mavedb:00000001-a-1") + self.user = Mock(id=5) + self.pipeline = Mock(spec=Pipeline) + self.entrypoint = _mock_entrypoint() + + factory_patch = patch("mavedb.lib.workflow.kickoff.PipelineFactory") + self.factory_cls = factory_patch.start() + self.factory = self.factory_cls.return_value + self.factory.create_pipeline.return_value = (self.pipeline, self.entrypoint) + yield + factory_patch.stop() + + async def _call(self, **kwargs): + return await enqueue_pipeline_for_score_set( + self.db, self.redis, pipeline_name="test_pipeline", score_set=self.score_set, user=self.user, **kwargs + ) + + async def test_returns_pipeline_and_job_when_enqueued(self): + job = Mock(spec=ArqJob) + self.redis.enqueue_job.return_value = job + + result = await self._call() + + assert result == (self.pipeline, job) + self.redis.enqueue_job.assert_awaited_once_with( + self.entrypoint.job_function, + self.entrypoint.id, + _job_id=f"{self.entrypoint.urn}#{self.entrypoint.retry_count}", + _queue_name=FAST_LANE_QUEUE_NAME, + ) + + async def test_returns_none_job_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + result = await self._call() + + assert result == (self.pipeline, None) + + async def test_propagates_key_error_for_unknown_pipeline_name(self): + self.factory.create_pipeline.side_effect = KeyError("unknown_pipeline") + + with pytest.raises(KeyError): + await self._call() + + async def test_discards_pipeline_and_reraises_when_enqueue_raises(self): + self.redis.enqueue_job.side_effect = ConnectionError("redis is unreachable") + + with pytest.raises(ConnectionError): + await self._call() + + self.factory.discard_pipeline.assert_called_once_with(self.pipeline) + + async def test_does_not_discard_pipeline_when_enqueue_succeeds(self): + self.redis.enqueue_job.return_value = Mock(spec=ArqJob) + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_does_not_discard_pipeline_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_default_correlation_id_built_from_pipeline_name_score_set_and_user(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value=None): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"].startswith(f"test_pipeline_{self.score_set.urn}_{self.user.id}_") + + async def test_correlation_id_from_context_used_when_present(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "context-correlation-id" + + async def test_extra_params_correlation_id_overrides_context(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call(extra_params={"correlation_id": "explicit-correlation-id"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "explicit-correlation-id" + + async def test_extra_params_merged_alongside_base_params(self): + await self._call(extra_params={"some_extra_param": "some_value"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["some_extra_param"] == "some_value" + assert params["score_set_id"] == self.score_set.id + assert params["updater_id"] == self.user.id + + async def test_create_pipeline_called_with_pipeline_name_and_user(self): + await self._call() + + call_kwargs = self.factory.create_pipeline.call_args.kwargs + assert call_kwargs["pipeline_name"] == "test_pipeline" + assert call_kwargs["creating_user"] is self.user + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetIntegration: + """Integration tests exercising the real PipelineFactory and a fakeredis-backed ARQ instance.""" + + @pytest.fixture(autouse=True) + def setup(self, session, setup_lib_db_with_score_set): + self.score_set = setup_lib_db_with_score_set + self.user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + async def test_creates_pipeline_and_job_run_records_and_enqueues_job( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, job = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert job is not None + assert session.execute(select(Pipeline).where(Pipeline.id == pipeline.id)).scalar_one() is pipeline + + job_run = session.execute( + select(JobRun).where(JobRun.pipeline_id == pipeline.id, JobRun.job_function == "process_data") + ).scalar_one() + assert job_run.job_params["required_param"] == "some_value" + + async def test_correlation_id_defaults_to_generated_value_without_request_context( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, _ = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline.correlation_id.startswith( + f"{sample_independent_pipeline_definition['name']}_{self.score_set.urn}_{self.user.id}_" + ) + + async def test_independent_calls_create_independent_pipelines_and_jobs( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline_one.id != pipeline_two.id + assert job_one is not None + assert job_two is not None + + async def test_returns_none_job_when_arq_job_id_collides( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with patch("mavedb.lib.workflow.kickoff.arq_job_id", return_value="fixed-dedupe-id"): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # ARQ deduplicates the second enqueue on the collided job id, but pipeline records are + # still created for both calls -- kickoff does not roll back on dedup. + assert job_one is not None + assert job_two is None + assert pipeline_one.id != pipeline_two.id + + async def test_unknown_pipeline_name_raises_key_error(self, session, arq_redis): + with pytest.raises(KeyError): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name="does_not_exist_pipeline", + score_set=self.score_set, + user=self.user, + ) + + async def test_discards_pipeline_records_when_enqueue_raises( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with ( + patch.object(arq_redis, "enqueue_job", AsyncMock(side_effect=ConnectionError("redis is unreachable"))), + pytest.raises(ConnectionError), + ): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # The pipeline (and its job runs) created before the failed enqueue must not be left behind. + assert session.execute(select(Pipeline)).scalars().all() == [] + assert session.execute(select(JobRun)).scalars().all() == [] diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/models/conftest.py b/tests/models/conftest.py new file mode 100644 index 000000000..5edfe06ee --- /dev/null +++ b/tests/models/conftest.py @@ -0,0 +1,80 @@ +import pytest + +from mavedb.models.enums import JobStatus +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.job_run import JobRun +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_EXPERIMENT, + TEST_EXPERIMENT_SET, + TEST_LICENSE, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, + TEST_USER, + VALID_EXPERIMENT_SET_URN, + VALID_EXPERIMENT_URN, + VALID_SCORE_SET_URN, +) + + +@pytest.fixture +def setup_lib_db_with_score_set(session, setup_lib_db): + """Build an experiment set, experiment, and score set on top of the base lib db (users/licenses).""" + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + experiment_set = ExperimentSet(**TEST_EXPERIMENT_SET, urn=VALID_EXPERIMENT_SET_URN) + experiment_set.created_by = user + experiment_set.modified_by = user + session.add(experiment_set) + session.commit() + session.refresh(experiment_set) + + experiment = Experiment(**TEST_EXPERIMENT, urn=VALID_EXPERIMENT_URN, experiment_set_id=experiment_set.id) + experiment.created_by = user + experiment.modified_by = user + session.add(experiment) + session.commit() + session.refresh(experiment) + + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, urn=VALID_SCORE_SET_URN, experiment_id=experiment.id, licence_id=TEST_LICENSE["id"] + ) + score_set.created_by = user + score_set.modified_by = user + session.add(score_set) + session.commit() + session.refresh(score_set) + + return score_set + + +@pytest.fixture +def setup_lib_db_with_variant(session, setup_lib_db_with_score_set): + """Add a single variant to the score set, for variant-subject event tests.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#1", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + return variant + + +@pytest.fixture +def job_run(session): + """Create a persisted JobRun to anchor an event's provenance.""" + job = JobRun( + job_type="test_annotation_job", + job_function="test_function", + status=JobStatus.RUNNING, + ) + session.add(job) + session.commit() + session.refresh(job) + return job diff --git a/tests/models/test_allele_hgvs.py b/tests/models/test_allele_hgvs.py new file mode 100644 index 000000000..c079feb00 --- /dev/null +++ b/tests/models/test_allele_hgvs.py @@ -0,0 +1,47 @@ +# ruff: noqa: E402 +"""``Allele.hgvs`` must mean the same thing on an instance and in SQL.""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.allele import Allele + +G, C, P = "NC_000017.11:g.1A>G", "NM_000001.1:c.2A>G", "NP_000001.1:p.Ala3Gly" + + +def test_instance_value_prefers_genomic_then_coding_then_protein(): + assert Allele(hgvs_g=G, hgvs_c=C, hgvs_p=P).hgvs == G + assert Allele(hgvs_c=C, hgvs_p=P).hgvs == C + assert Allele(hgvs_p=P).hgvs == P + + +def test_instance_value_is_none_when_nothing_is_set(): + assert Allele().hgvs is None + + +@pytest.mark.integration +def test_the_sql_expression_matches_the_allele_at_every_level(session): + """Regression: a hybrid with no SQL expression compiled ``Allele.hgvs`` to ``hgvs_g`` alone, so a query + for a coding or protein allele silently matched nothing. + """ + alleles = { + G: Allele(vrs_digest="g", level="genomic", hgvs_g=G), + C: Allele(vrs_digest="c", level="cdna", hgvs_c=C), + P: Allele(vrs_digest="p", level="protein", hgvs_p=P), + } + session.add_all(alleles.values()) + session.commit() + + for hgvs, allele in alleles.items(): + assert session.scalars(select(Allele.id).where(Allele.hgvs == hgvs)).all() == [allele.id] + + +@pytest.mark.integration +def test_transcript_still_derives_from_the_same_expression(session): + session.add(Allele(vrs_digest="c", level="cdna", hgvs_c=C)) + session.commit() + + assert session.scalars(select(Allele.vrs_digest).where(Allele.transcript == "NM_000001.1")).all() == ["c"] diff --git a/tests/models/test_annotation_event_model.py b/tests/models/test_annotation_event_model.py new file mode 100644 index 000000000..336536479 --- /dev/null +++ b/tests/models/test_annotation_event_model.py @@ -0,0 +1,103 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy.exc import IntegrityError + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +@pytest.fixture +def allele(session): + allele = Allele(vrs_digest="test-variant-event-allele-digest", level="genomic") + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _expect_rejected(session, event): + """Adding `event` must violate a CHECK constraint on flush.""" + session.add(event) + with pytest.raises(IntegrityError): + session.flush() + session.rollback() + + +class TestSubjectConstraint: + def test_variant_subject_event_inserts(self, session, setup_lib_db_with_variant, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + score_set_id=setup_lib_db_with_variant.score_set_id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.allele_id is None + + def test_allele_subject_event_inserts(self, session, allele, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason="no_record", + source_version="4.1.0", + job_run_id=job_run.id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.variant_id is None + + def test_variant_subject_type_with_allele_id_rejected(self, session, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_allele_subject_type_with_variant_id_rejected(self, session, setup_lib_db_with_variant): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="created", + ), + ) + + def test_neither_subject_set_rejected(self, session): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_both_subjects_set_rejected(self, session, setup_lib_db_with_variant, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) diff --git a/tests/models/test_annotation_event_view.py b/tests/models/test_annotation_event_view.py new file mode 100644 index 000000000..14c74f802 --- /dev/null +++ b/tests/models/test_annotation_event_view.py @@ -0,0 +1,147 @@ +# ruff: noqa: E402 +"""Tests for the v_current_annotation_events view (current event per subject + type).""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.annotation_event_view import CurrentAnnotationEventView +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +def _allele(session, digest, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _event(session, job_run, annotation_type, disposition, reason, *, allele_id=None, variant_id=None, **kw): + event = AnnotationEvent( + annotation_type=annotation_type, + allele_id=allele_id, + variant_id=variant_id, + disposition=disposition, + reason=reason, + job_run_id=job_run.id, + **kw, + ) + session.add(event) + session.commit() + session.refresh(event) + return event + + +def _view_rows(session, **filters): + stmt = select(CurrentAnnotationEventView) + for col, val in filters.items(): + stmt = stmt.where(getattr(CurrentAnnotationEventView, col) == val) + return list(session.scalars(stmt).all()) + + +def test_returns_only_latest_event_per_subject_and_type(session, job_run): + allele = _allele(session, "view-latest") + _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.ABSENT, "no_record", allele_id=allele.id + ) + latest = _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.PRESENT, "created", allele_id=allele.id + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].disposition == Disposition.PRESENT + + +def test_clinvar_is_multi_live_one_row_per_release(session, job_run): + allele = _allele(session, "view-clinvar") + e_2025 = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2025", + ) + _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2026", + ) + e_2026_latest = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "superseded", + allele_id=allele.id, + source_version="01_2026", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.CLINVAR_CONTROL.value) + + by_version = {r.source_version: r.id for r in rows} + assert by_version == {"01_2025": e_2025.id, "01_2026": e_2026_latest.id} + + +def test_non_clinvar_collapses_across_versions(session, job_run): + """gnomAD/VEP supersede to a single current state: a re-fetch at a new source_version yields one + row, not one per version (the CASE folds source_version in only for ClinVar).""" + allele = _allele(session, "view-gnomad-version") + _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="4.0.0", + ) + latest = _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "reconfirmed", + allele_id=allele.id, + source_version="4.1.0", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].source_version == "4.1.0" + + +def test_variant_subject_events_present_and_keyed_by_variant(session, setup_lib_db_with_variant, job_run): + variant = setup_lib_db_with_variant + _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.FAILED, "failed", variant_id=variant.id) + latest = _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.PRESENT, "mapped", variant_id=variant.id) + + rows = _view_rows(session, variant_id=variant.id, annotation_type=AnnotationType.VRS_MAPPING.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].allele_id is None + + +def test_distinct_subjects_yield_distinct_rows(session, job_run): + a1 = _allele(session, "view-a1") + a2 = _allele(session, "view-a2") + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.PRESENT, "created", allele_id=a1.id) + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.NOT_APPLICABLE, "no_hgvs", allele_id=a2.id) + + rows = _view_rows(session, annotation_type=AnnotationType.CLINGEN_ALLELE_ID.value) + + assert {r.allele_id for r in rows} == {a1.id, a2.id} diff --git a/tests/models/test_variant.py b/tests/models/test_variant.py new file mode 100644 index 000000000..81da770e5 --- /dev/null +++ b/tests/models/test_variant.py @@ -0,0 +1,12 @@ +import pytest + +from mavedb.models.variant import Variant + + +@pytest.mark.unit +@pytest.mark.parametrize( + "urn, expected", + [("urn:mavedb:00000053-a-1#648022", 648022), ("tmp:0c8d1a2e-5f3b-4c6d-9e7f-1a2b3c4d5e6f#1", 1), (None, None)], +) +def test_variant_number_is_the_integer_after_the_hash(urn, expected): + assert Variant(urn=urn).variant_number == expected diff --git a/tests/models/test_variant_annotation_view.py b/tests/models/test_variant_annotation_view.py new file mode 100644 index 000000000..d1c2f1d72 --- /dev/null +++ b/tests/models/test_variant_annotation_view.py @@ -0,0 +1,115 @@ +# ruff: noqa: E402 +"""Tests for the v_variant_annotations view after its rewrite onto the MappingRecord/Allele substrate. + +The view is a flat operator convenience projection: one row per (variant, current annotation_type). It +reconstructs the legacy ``mapped_variants`` row's flat ``hgvs_g``/``hgvs_c``/``hgvs_p`` triple by +pivoting the record's live allele links by level, and carries the single-valued ``clingen_allele_id`` +and VEP consequence from the record's authoritative allele. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.variant import Variant +from mavedb.models.variant_annotation_view import VariantAnnotationView +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +def _rows_for(session, variant_urn): + # Select explicit columns (not the whole entity): the view is keyless, so an all-null-identity row + # (an unmapped variant) would materialize as a single ``None`` entity. Column rows sidestep that. + stmt = select( + VariantAnnotationView.mapping_record_id, + VariantAnnotationView.hgvs_assay_level, + VariantAnnotationView.hgvs_g, + VariantAnnotationView.hgvs_c, + VariantAnnotationView.hgvs_p, + VariantAnnotationView.clingen_allele_id, + VariantAnnotationView.vep_functional_consequence, + ).where(VariantAnnotationView.variant_urn == variant_urn) + return list(session.execute(stmt).all()) + + +def test_reconstructs_flat_triple_and_authoritative_fields(session, setup_lib_db_with_variant): + """A cdna-assay record with genomic/cdna/protein alleles fills all three HGVS slots; the + authoritative (cdna) allele supplies clingen_allele_id and the VEP consequence.""" + variant = setup_lib_db_with_variant + record = seed_mapping_record( + session, + variant, + assay_level="cdna", + hgvs_assay_level="NM_000001.1:c.1A>T", + alleles=[ + AlleleSpec( + digest="d-cdna", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + hgvs_c="NM_000001.1:c.1A>T", + vep_consequence="missense_variant", + ), + AlleleSpec(digest="d-genomic", level="genomic", hgvs_g="NC_000001.11:g.100A>T"), + AlleleSpec(digest="d-protein", level="protein", hgvs_p="NP_000001.1:p.Met1Leu"), + ], + ) + + rows = _rows_for(session, variant.urn) + assert len(rows) == 1 + row = rows[0] + assert row.mapping_record_id == record.id + assert row.hgvs_assay_level == "NM_000001.1:c.1A>T" + assert row.hgvs_g == "NC_000001.11:g.100A>T" + assert row.hgvs_c == "NM_000001.1:c.1A>T" + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.clingen_allele_id == "CA123" + assert row.vep_functional_consequence == "missense_variant" + + +def test_protein_assay_leaves_nucleotide_slots_null(session, setup_lib_db_with_variant): + """A protein-assay record has only a protein allele: hgvs_p is filled, the nucleotide slots stay + null, and the authoritative protein allele still supplies clingen_allele_id.""" + variant = setup_lib_db_with_variant + seed_mapping_record( + session, + variant, + assay_level="protein", + hgvs_assay_level="NP_000001.1:p.Met1Leu", + alleles=[ + AlleleSpec( + digest="d-prot-only", + level="protein", + is_authoritative=True, + clingen_allele_id="PA9", + hgvs_p="NP_000001.1:p.Met1Leu", + ), + ], + ) + + row = _rows_for(session, variant.urn)[0] + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.clingen_allele_id == "PA9" + + +def test_unmapped_variant_retained_with_null_mapping(session, setup_lib_db_with_score_set): + """The outer join keeps a variant with no live mapping record — it still appears with a null + mapping_record_id and empty mapped fields.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#7", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + row = _rows_for(session, variant.urn)[0] + assert row.mapping_record_id is None + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.hgvs_p is None + assert row.clingen_allele_id is None + assert row.vep_functional_consequence is None diff --git a/tests/routers/conftest.py b/tests/routers/conftest.py index ba34c5489..0971bde47 100644 --- a/tests/routers/conftest.py +++ b/tests/routers/conftest.py @@ -3,7 +3,7 @@ import pytest -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.enums.user_role import UserRole @@ -52,8 +52,8 @@ def setup_router_db(session): db.add(License(**TEST_INACTIVE_LICENSE)) db.add(License(**EXTRA_LICENSE)) db.add(Contributor(**EXTRA_CONTRIBUTOR)) - db.add(ClinicalControl(**TEST_CLINVAR_CONTROL)) - db.add(ClinicalControl(**TEST_GENERIC_CLINICAL_CONTROL)) + db.add(ClinvarControl(**TEST_CLINVAR_CONTROL)) + db.add(ClinvarControl(**TEST_GENERIC_CLINICAL_CONTROL)) db.add(GnomADVariant(**TEST_GNOMAD_VARIANT)) db.bulk_save_objects([ControlledKeyword(**keyword_obj) for keyword_obj in TEST_DB_KEYWORDS]) db.commit() diff --git a/tests/routers/test_allele.py b/tests/routers/test_allele.py new file mode 100644 index 000000000..bf995552c --- /dev/null +++ b/tests/routers/test_allele.py @@ -0,0 +1,293 @@ +# ruff: noqa: E402 +"""Router tests for the allele-detail endpoints (``GET /alleles/{digest}`` and ``?clingenAlleleId=``). + +Exercise the HTTP surface end to end: the envelope serializes and validates, the equivalence class is +labelled relative to the focus, CAID fetch focuses the nt-canonical change, ``as_of`` is echoed and +reconstructs membership, an unknown id is a plain 404, and an allele reachable only through score sets +the caller may not read is the same 404, with the equivalence class scoped to readable score sets. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant +from mavedb.view_models.allele_detail import AlleleDetail +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _ir(tag: str) -> str: + """A valid GA4GH IR (``ga4gh:VA.`` + exactly 32 base64url chars), tagged for readability.""" + return f"ga4gh:VA.{tag.ljust(32, '0')}" + + +CDNA = _ir("cdna") +GEN = _ir("gen") +PROT = _ir("prot") +COUSIN = _ir("cousin") + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed(session, variant_urn): + """Coding-measured record: authoritative cdna (+ClinGen CA1 +VEP), its genomic partner (also CA1), + the protein apex, and a synonymous cousin in its own projection group.""" + measured = GroupSpec( + hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p="NP_000537.3:p.Ala406Thr" + ) + seed_mapping_record( + session, + variant_urn, + alleles=[ + AlleleSpec( + digest=CDNA, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + group=measured, + ), + AlleleSpec( + digest=GEN, + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + group=measured, + ), + AlleleSpec( + digest=PROT, + level="protein", + clingen_allele_id="PA1", + hgvs_p="NP_000537.3:p.Ala406Thr", + post_mapped=_post_mapped(), + ), + AlleleSpec( + digest=COUSIN, + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + group=GroupSpec(hgvs_c="NM_000546.6:c.1218C>T"), + ), + ], + assay_level="cdna", + ) + + +def test_get_allele_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + + assert body["digest"] == CDNA + assert body["level"] == "cdna" + assert body["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["clingenAlleleId"] == "CA1" + assert body["vrs"]["type"] == "Allele" + # Full equivalence class, keyed by digest, labelled relative to the queried allele (camelCase). + assert set(body["alleles"]) == {CDNA, GEN, PROT, COUSIN} + assert body["alleles"][CDNA]["isFocus"] is True + assert "derivation" not in body["alleles"][CDNA] # focus carries none; dropped by exclude_none + assert body["alleles"][GEN]["relation"] == "coordinate_representation_of" + assert body["alleles"][GEN]["derivation"] == "projection" + assert body["alleles"][PROT]["relation"] == "translation_of" + assert body["alleles"][COUSIN]["relation"] == "co_encodes" + assert body["alleles"][COUSIN]["derivation"] == "convergent" + # Annotations share keys with the class and join by digest. + assert set(body["annotations"]) == {CDNA, GEN, PROT, COUSIN} + assert body["annotations"][CDNA]["vep"]["consequence"] == "missense_variant" + # Measurement-agnostic: no variant-detail measurement fields. + assert "molecularRepresentation" not in body + assert "classifications" not in body + assert "isCurrent" not in body + assert response.headers["X-As-Of"] == "current" + + +def test_get_allele_detail_by_caid(client, session, data_provider, data_files, setup_router_db): + """CAID fetch on the overloaded `/alleles/{identifier}` path focuses the nt-canonical change (both + genomic + coding frames isFocus).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/CA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "CA1" + assert body["alleles"][CDNA]["isFocus"] is True + assert body["alleles"][GEN]["isFocus"] is True # the CAID's other frame + assert body["alleles"][COUSIN]["derivation"] == "convergent" + + +def test_get_allele_detail_by_paid(client, session, data_provider, data_files, setup_router_db): + """PAID fetch focuses the protein allele; its nucleotide equivalents surface as candidates — the + same `/alleles/{identifier}` path, no separate route or param.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/PA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "PA1" + assert body["alleles"][PROT]["isFocus"] is True + # Walking down from the protein is ambiguous: every nucleotide member is a candidate. + for nt in (CDNA, GEN, COUSIN): + assert body["alleles"][nt]["derivation"] == "candidate", nt + + +def test_get_allele_detail_unknown_digest_is_404(client, setup_router_db): + response = client.get(f"/api/v1/alleles/{_ir('nonexistent')}") + assert response.status_code == 404 + + +def test_get_allele_detail_unknown_caid_is_404(client, setup_router_db): + response = client.get("/api/v1/alleles/CA999999") + assert response.status_code == 404 + + +@pytest.mark.parametrize("identifier", [CDNA, "CA1", "PA1"]) +@pytest.mark.parametrize("anonymous", [True, False]) +def test_get_allele_detail_is_404_when_only_unreadable_score_sets_link_it( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides, identifier, anonymous +): + """An allele exists only because of a private score set the caller may not read, so it is not found.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + if anonymous: + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/alleles/{identifier}") + else: + response = client.get(f"/api/v1/alleles/{identifier}") + + assert response.status_code == 404 + + +def test_get_allele_detail_is_404_for_an_allele_no_score_set_links(client, session, setup_router_db): + session.add(Allele(vrs_digest=CDNA, level="cdna", hgvs_c="NM_000546.6:c.1216G>A", post_mapped=_post_mapped())) + session.commit() + + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 404 + + +def test_get_allele_detail_scopes_equivalence_class_to_readable_score_sets( + client, session, data_provider, data_files, setup_router_db +): + """A shared allele is served through the readable score set, but members co-linked only by an + unreadable score set's record are withheld.""" + experiment = create_experiment(client) + private_score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{private_score_set['urn']}#1") + change_ownership(session, private_score_set["urn"], ScoreSetDbModel) + + readable_score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(Variant).where(Variant.urn == f"{readable_score_set['urn']}#1")) + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) + session.add(record) + session.commit() + for digest, is_authoritative in ((CDNA, True), (PROT, False)): + allele = session.scalar(select(Allele).where(Allele.vrs_digest == digest)) + session.add( + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=allele.id, + is_authoritative=is_authoritative, + ) + ) + session.commit() + + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + body = response.json() + assert set(body["alleles"]) == {CDNA, PROT} + assert set(body["annotations"]) == {CDNA, PROT} + # The c↔g pairing lives only on the unreadable record. + assert "projectionOf" not in body["alleles"][CDNA] + + # The genomic frame of CA1 is reachable only through the unreadable record, so a CAID fetch + # focuses the coding frame alone. + caid_response = client.get("/api/v1/alleles/CA1") + assert caid_response.status_code == 200 + assert set(caid_response.json()["alleles"]) == {CDNA, PROT} + + +def test_get_allele_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + """``as_of`` is echoed and reconstructs membership: before the record was live the class collapses + to the focus (the allele row itself is content-addressed, so it still resolves).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert response.status_code == 200 + assert response.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert set(response.json()["alleles"]) == {CDNA} diff --git a/tests/routers/test_calibration_controls.py b/tests/routers/test_calibration_controls.py new file mode 100644 index 000000000..98ca322ec --- /dev/null +++ b/tests/routers/test_calibration_controls.py @@ -0,0 +1,732 @@ +# ruff: noqa: E402 + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +import json +from unittest.mock import patch + +from sqlalchemy import delete, select + +from mavedb.lib.validation.constants.general import ( + calibration_control_status_column_name, + calibration_variant_column_name, +) +from mavedb.models.calibration_control import CalibrationControl +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.score_calibration import ScoreCalibration as CalibrationDbModel +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_PUBMED_IDENTIFIER, +) +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import ( + create_test_score_calibration_in_score_set_via_client, + publish_test_score_calibration_via_client, +) +from tests.helpers.util.score_set import create_seq_score_set_with_mapped_variants, publish_score_set + +CALIBRATION_PUBLICATIONS = [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +def _score_set_with_variant_urns(client, session, data_provider, data_files): + """Create a score set with mapped variants; return (score_set dict, list of its variant URNs).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # Calibrations on a private score set cannot be published. Publishing also renames the variant URNs, + # so it must precede the URN lookup below. + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + score_set_orm = session.query(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"]).one() + variant_urns = [ + variant.urn + for variant in session.scalars(select(Variant).where(Variant.score_set_id == score_set_orm.id)).all() + ] + return score_set, variant_urns + + +def _calibration_payload(score_set_urn, **extra): + return {**deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), "scoreSetUrn": score_set_urn, **extra} + + +CONTROLS_CSV_HEADER = f"{calibration_variant_column_name},{calibration_control_status_column_name}" + + +def _create_private_calibration(client, session, data_provider, data_files): + """Create a score set with variants and a private range-based calibration; return the calibration dict.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, + session, + data_provider, + experiment["urn"], + data_files / "scores.csv", + ) + # Calibrations on a private score set cannot be published, so publish the score set first. + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + return create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + +def _attach_control(session, calibration_urn, controls_not_phi): + """Attach one control to a calibration at the ORM level (endpoint wiring is #753) and set the PHI flag.""" + calibration = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration_urn).one() + variant = session.scalars(select(Variant).where(Variant.score_set_id == calibration.score_set_id)).first() + + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=calibration.created_by, + modified_by=calibration.created_by, + ) + ) + calibration.controls_not_phi = controls_not_phi + session.add(calibration) + session.commit() + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_publish_calibration_with_controls_when_phi_affirmed( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + calibration = _create_private_calibration(client, session, data_provider, data_files) + _attach_control(session, calibration["urn"], controls_not_phi=True) + + response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/publish") + + assert response.status_code == 200 + assert response.json()["private"] is False + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_publish_calibration_without_controls_regardless_of_phi_flag( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + calibration = _create_private_calibration(client, session, data_provider, data_files) + + # No controls attached, but an explicit (non-True) PHI flag must not block publishing. + item = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + item.controls_not_phi = False + session.add(item) + session.commit() + + response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/publish") + + assert response.status_code == 200 + assert response.json()["private"] is False + + +########################################################### +# Controls create / update / read wiring (#753) +########################################################### + + +def _create_with_controls(client, score_set_urn, controls, **extra): + response = client.post( + "/api/v1/score-calibrations/", + json=_calibration_payload(score_set_urn, controls=controls, **{"controlsNotPhi": True, **extra}), + ) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_with_inline_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + + calibration = _create_with_controls( + client, + score_set["urn"], + [ + {"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}, + {"variantUrn": variant_urns[1], "clinicalStatus": "benign"}, + ], + ) + + assert {(c["variantUrn"], c["clinicalStatus"]) for c in calibration["controls"]} == { + (variant_urns[0], "pathogenic"), + (variant_urns[1], "benign"), + } + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_with_controls_file( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + csv = f"{CONTROLS_CSV_HEADER}\n{variant_urns[0]},pathogenic\n{variant_urns[1]},Benign\n" + + response = client.post( + "/api/v1/score-calibrations/", + data={"calibration_json": json.dumps(_calibration_payload(score_set["urn"], controlsNotPhi=True))}, + files={"controls_file": ("controls.csv", csv, "text/csv")}, + ) + + assert response.status_code == 200, response.text + assert {(c["variantUrn"], c["clinicalStatus"]) for c in response.json()["controls"]} == { + (variant_urns[0], "pathogenic"), + (variant_urns[1], "benign"), + } + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_rejects_inline_controls_and_controls_file_together( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + csv = f"{CONTROLS_CSV_HEADER}\n{variant_urns[0]},pathogenic\n" + + response = client.post( + "/api/v1/score-calibrations/", + data={ + "calibration_json": json.dumps( + _calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "benign"}] + ) + ) + }, + files={"controls_file": ("controls.csv", csv, "text/csv")}, + ) + + assert response.status_code == 422 + assert "not both" in str(response.json()["detail"]) + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_get_calibration_returns_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.get(f"/api/v1/score-calibrations/{calibration['urn']}") + + assert response.status_code == 200 + assert response.json()["controls"][0]["variantUrn"] == variant_urns[0] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_list_endpoint_reports_controls_count_without_full_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + _create_with_controls( + client, + score_set["urn"], + [ + {"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}, + {"variantUrn": variant_urns[1], "clinicalStatus": "benign"}, + ], + ) + + response = client.get(f"/api/v1/score-calibrations/score-set/{score_set['urn']}") + + assert response.status_code == 200, response.text + item = response.json()[0] + # List responses carry the count, not the full controls list. + assert item["controlsCount"] == 2 + assert "controls" not in item + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_replaces_controls(client, setup_router_db, mock_publication_fetch, session, data_provider, data_files): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}], + controlsNotPhi=True, + ), + ) + + assert response.status_code == 200, response.text + controls = response.json()["controls"] + assert len(controls) == 1 + assert controls[0]["variantUrn"] == variant_urns[1] + assert controls[0]["clinicalStatus"] == "benign" + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_with_empty_controls_clears_them( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], controls=[]), + ) + + assert response.status_code == 200, response.text + assert response.json()["controls"] == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_without_controls_leaves_them_unchanged( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + # The payload omits controls entirely, so existing controls must be preserved. + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", json=_calibration_payload(score_set["urn"]) + ) + + assert response.status_code == 200, response.text + assert len(response.json()["controls"]) == 1 + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_replace_controls_without_reaffirming( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + # The prior affirmation covered the old controls, not the new ones. + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}] + ), + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls_not_phi is True + assert [control.variant.urn for control in refreshed.controls] == [variant_urns[0]] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_modify_controls_with_affirmation_keeps_phi_flag( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + # Re-affirming in the same request that changes controls keeps the flag set. + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}], + controlsNotPhi=True, + ), + ) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is True + + +########################################################### +# PHI affirmation invariant: controls are stored only while affirmed +########################################################### + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +@pytest.mark.parametrize("controls_not_phi", [None, False]) +def test_cannot_create_calibration_with_inline_controls_without_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, controls_not_phi +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + + response = client.post( + "/api/v1/score-calibrations/", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=controls_not_phi, + ), + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + assert session.scalars(select(CalibrationControl)).all() == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_create_calibration_with_controls_file_without_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + csv = f"{CONTROLS_CSV_HEADER}\n{variant_urns[0]},pathogenic\n" + + response = client.post( + "/api/v1/score-calibrations/", + data={"calibration_json": json.dumps(_calibration_payload(score_set["urn"]))}, + files={"controls_file": ("controls.csv", csv, "text/csv")}, + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + assert session.scalars(select(CalibrationControl)).all() == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_create_calibration_without_controls_or_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, _ = _score_set_with_variant_urns(client, session, data_provider, data_files) + + response = client.post("/api/v1/score-calibrations/", json=_calibration_payload(score_set["urn"])) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is None + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +@pytest.mark.parametrize("controls_not_phi", [None, False]) +def test_withdrawing_phi_affirmation_deletes_controls( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, controls_not_phi +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [ + {"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}, + {"variantUrn": variant_urns[1], "clinicalStatus": "benign"}, + ], + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], controlsNotPhi=controls_not_phi), + ) + + assert response.status_code == 200, response.text + assert response.json()["controls"] == [] + assert response.json()["controlsNotPhi"] is controls_not_phi + assert session.scalars(select(CalibrationControl)).all() == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_submit_controls_while_withdrawing_phi_affirmation( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}], + controlsNotPhi=False, + ), + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + + # A rejected update stages nothing: the original controls and affirmation stand. + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls_not_phi is True + assert [control.variant.urn for control in refreshed.controls] == [variant_urns[0]] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_create_calibration_ignores_client_supplied_private_field( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + # The create schema has no `private` field — creation always starts private, and publishing is the + # only path to making one public. + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + + response = client.post( + "/api/v1/score-calibrations/", + json={ + **_calibration_payload(score_set["urn"]), + "private": False, + "controls": [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + "controlsNotPhi": True, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["private"] is True + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_add_controls_to_published_calibration_without_phi_affirmation_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ), + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + + # The gate must not have attached the controls. + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_add_controls_to_published_calibration_with_phi_affirmation_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], + controls=[{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ), + ) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is True + assert len(response.json()["controls"]) == 1 + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_cannot_replace_controls_on_published_calibration_without_reaffirming_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + # The invariant holds for published calibrations and for admins alike. + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload( + score_set["urn"], controls=[{"variantUrn": variant_urns[1], "clinicalStatus": "benign"}] + ), + ) + + assert response.status_code == 422 + assert "protected health information" in str(response.json()["detail"]).lower() + + # The gate must not have replaced the controls or reset the affirmation. + refreshed = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + assert refreshed.controls_not_phi is True + assert len(refreshed.controls) == 1 + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_withdrawing_phi_affirmation_on_published_calibration_deletes_controls_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, score_set["urn"], [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}] + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], controlsNotPhi=False), + ) + + assert response.status_code == 200, response.text + assert response.json()["private"] is False + assert response.json()["controls"] == [] + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_can_update_published_calibration_leaving_affirmed_controls_untouched_as_admin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, admin_app_overrides +): + score_set, variant_urns = _score_set_with_variant_urns(client, session, data_provider, data_files) + calibration = _create_with_controls( + client, + score_set["urn"], + [{"variantUrn": variant_urns[0], "clinicalStatus": "pathogenic"}], + controlsNotPhi=True, + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + # An edit that neither submits controls nor touches controlsNotPhi must leave the standing + # affirmation alone, so it should not trip the gate. + with DependencyOverrider(admin_app_overrides): + response = client.put( + f"/api/v1/score-calibrations/{calibration['urn']}", + json=_calibration_payload(score_set["urn"], notes="Updated notes"), + ) + + assert response.status_code == 200, response.text + assert response.json()["controlsNotPhi"] is True + assert len(response.json()["controls"]) == 1 + + +########################################################### +# Control placement (functional_classification_id) — Option A +########################################################### + + +def _add_control_on_variant(session, calibration, variant, status=CalibrationControlStatus.pathogenic): + """Attach one control to ``calibration`` on a chosen ``variant`` at the ORM level.""" + session.add( + CalibrationControl( + calibration=calibration, + variant=variant, + clinical_status=status, + created_by=calibration.created_by, + modified_by=calibration.created_by, + ) + ) + session.commit() + + +def _placement_by_variant_id(calibration): + """Ground-truth ``{variant_id: functional_classification_id}`` from stored bin membership.""" + placement = {} + for classification in calibration.functional_classifications: + for variant in classification.variants: + placement[variant.id] = classification.id + return placement + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_get_calibration_populates_control_placement( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + """A control whose variant a range files reports that range's id as its placement.""" + calibration = _create_private_calibration(client, session, data_provider, data_files) + calibration_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + + placement = _placement_by_variant_id(calibration_orm) + assert placement, "fixture calibration should file at least one variant into a range" + + variant_id, expected_classification_id = next(iter(placement.items())) + variant = session.get(Variant, variant_id) + _add_control_on_variant(session, calibration_orm, variant) + + response = client.get(f"/api/v1/score-calibrations/{calibration['urn']}") + assert response.status_code == 200, response.text + + control = next(c for c in response.json()["controls"] if c["variantUrn"] == variant.urn) + assert control["functionalClassificationId"] == expected_classification_id + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_control_placement_is_none_when_variant_has_no_bin( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + """A control whose variant is filed under none of the calibration's ranges reports null placement.""" + calibration = _create_private_calibration(client, session, data_provider, data_files) + calibration_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + + variant = session.scalars(select(Variant).where(Variant.score_set_id == calibration_orm.score_set_id)).first() + _add_control_on_variant(session, calibration_orm, variant) + + # Strip any bin membership for this variant so its placement must resolve to None. + association = score_calibration_functional_classification_variants_association_table + session.execute(delete(association).where(association.c.variant_id == variant.id)) + session.commit() + + response = client.get(f"/api/v1/score-calibrations/{calibration['urn']}") + assert response.status_code == 200, response.text + + control = next(c for c in response.json()["controls"] if c["variantUrn"] == variant.urn) + assert control["functionalClassificationId"] is None + + +@pytest.mark.parametrize("mock_publication_fetch", [CALIBRATION_PUBLICATIONS], indirect=["mock_publication_fetch"]) +def test_control_placement_scoped_to_own_calibration( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + """Placement reflects the control's own calibration, not another that happens to bin the variant.""" + calibration = _create_private_calibration(client, session, data_provider, data_files) + calibration_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == calibration["urn"]).one() + + # Capture plain ids/urns up front; creating the second calibration commits and expires ORM objects. + score_set_urn = calibration_orm.score_set.urn + placement = _placement_by_variant_id(calibration_orm) + assert placement + binned_variant_id = next(iter(placement)) + binned_variant_urn = session.get(Variant, binned_variant_id).urn + + # A second calibration on the same score set bins the same variant identically; clear only the + # second's membership for it, so the first calibration's binning must not leak into the second's + # placement. + second = create_test_score_calibration_in_score_set_via_client( + client, score_set_urn, deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + second_orm = session.query(CalibrationDbModel).where(CalibrationDbModel.urn == second["urn"]).one() + + association = score_calibration_functional_classification_variants_association_table + second_classification_ids = [fc.id for fc in second_orm.functional_classifications] + session.execute( + delete(association).where( + association.c.variant_id == binned_variant_id, + association.c.functional_classification_id.in_(second_classification_ids), + ) + ) + _add_control_on_variant(session, second_orm, session.get(Variant, binned_variant_id)) + + response = client.get(f"/api/v1/score-calibrations/{second['urn']}") + assert response.status_code == 200, response.text + + control = next(c for c in response.json()["controls"] if c["variantUrn"] == binned_variant_urn) + assert control["functionalClassificationId"] is None diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py new file mode 100644 index 000000000..8569465f1 --- /dev/null +++ b/tests/routers/test_clingen_allele.py @@ -0,0 +1,332 @@ +# ruff: noqa: E402 +"""Router tests for the ClinGen-allele measurements endpoint +(``GET /clingen-alleles/{caid}/measurements``). + +Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` +content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded +measurements and research-use-only calls are opt-in, a private score set's measurement never leaks, and the response body's +order reflects the ranked sort (current-then-superseded, direct-then-related, newest-published, urn). +""" + +from datetime import date + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.lib.mondo import get_generic_disease_term +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.user import User +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.view_models.allele_measurement import AlleleMeasurement +from tests.helpers.constants import TEST_USER +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + + +def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123", vrs_digest="cdna-digest"): + """Give a variant a live coding-measured mapping record whose authoritative allele carries the + ClinGen id — the minimal shape for a direct measurement on that CA's page. ``vrs_digest`` is + content-unique (DB constraint), so pass a distinct one when seeding more than one per test.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest=vrs_digest, level="cdna", clingen_allele_id=clingen_allele_id) + session.add(measured) + session.commit() + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=measured.id, is_authoritative=True + ) + ) + session.commit() + + +def test_measurements_serialize(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_cdna_measurement(session, urn) + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + AlleleMeasurement.model_validate(body[0]) + assert body[0]["variantUrn"] == urn + assert body[0]["relationship"] == "direct" + assert body[0]["assayLevel"] == "cdna" + assert body[0]["assayLevelHgvs"] == "NM_000546.6:c.1216G>A" + assert body[0]["submittedHgvs"] == "c.1A>T" + assert body[0]["scoreSetUrn"] == score_set["urn"] + assert body[0]["isCurrent"] is True + assert "supersededByScoreSet" not in body[0] # dropped by exclude_none when current + assert response.headers["X-As-Of"] == "current" + + +def test_research_use_only_call_is_opt_in(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_cdna_measurement(session, urn) + + user = session.scalar(select(User).where(User.username == TEST_USER["username"])) + calibration = ScoreCalibration( + score_set_id=session.scalar(select(ScoreSetDbModel.id).where(ScoreSetDbModel.urn == score_set["urn"])), + title="Research calibration", + primary=False, + private=False, + research_use_only=True, + disease_term=get_generic_disease_term(session), + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + classification = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="abnormal", + functional_classification="abnormal", + range=[None, 0.0], + inclusive_lower_bound=False, + ) + classification.variants = [session.scalar(select(VariantDbModel).where(VariantDbModel.urn == urn))] + session.add(classification) + session.commit() + + default = client.get("/api/v1/clingen-alleles/CA123/measurements").json() + assert "classification" not in default[0] + assert default[0]["classificationIsResearchUseOnly"] is False + + opted = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_research_use_only": True}).json() + assert opted[0]["classification"]["functionalClassification"] == "abnormal" + assert opted[0]["classificationIsResearchUseOnly"] is True + + +def test_unknown_clingen_id_returns_empty_list(client, setup_router_db): + response = client.get("/api/v1/clingen-alleles/CA000/measurements") + assert response.status_code == 200 + assert response.json() == [] + + +def test_superseded_measurement_is_opt_in(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1") + + default = client.get("/api/v1/clingen-alleles/CA123/measurements") + assert default.status_code == 200 + assert default.json() == [] + + opted = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + assert opted.status_code == 200 + body = opted.json() + assert len(body) == 1 + assert body[0]["isCurrent"] is False + assert body[0]["supersededByScoreSet"] == newer["urn"] + + +def test_nucleotide_siblings_shown(client, session, data_provider, data_files, setup_router_db): + """A different DNA variant encoding the same protein consequence surfaces on a CA page as a + ``nucleotide_encoding``, reached through the shared protein consequence.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + # Two coding measurements encoding the same protein consequence (PA9), each carrying its own CA. + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=nt.id, is_authoritative=True + ) + ) + session.add( + MappingRecordAllele(mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=prot.id) + ) + session.commit() + + response = client.get("/api/v1/clingen-alleles/CA111/measurements") + assert response.status_code == 200 + by_urn = {m["variantUrn"]: m for m in response.json()} + assert set(by_urn) == {f"{score_set['urn']}#1", f"{score_set['urn']}#2"} + assert by_urn[f"{score_set['urn']}#1"]["relationship"] == "direct" + assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" + + +def test_direct_measurements_sort_before_related(client, session, data_provider, data_files, setup_router_db): + """The list is ordered, and direct measurements precede related ones in the body: a CA query returns + the directly-assayed change first, then the sibling nucleotide encoding.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add( + MappingRecordAllele( + mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=nt.id, is_authoritative=True + ) + ) + session.add( + MappingRecordAllele(mapping_record_id=record.id, score_set_id=record.score_set_id, allele_id=prot.id) + ) + session.commit() + + response = client.get("/api/v1/clingen-alleles/CA111/measurements") + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["relationship"]) for m in body] == [ + (f"{score_set['urn']}#1", "direct"), + (f"{score_set['urn']}#2", "nucleotide_encoding"), + ] + + +def test_current_measurements_sort_before_superseded(client, session, data_provider, data_files, setup_router_db): + """With superseded measurements opted in, current ones sort ahead of superseded ones in the body.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1", vrs_digest="cdna-older") + _seed_cdna_measurement(session, f"{newer['urn']}#1", vrs_digest="cdna-newer") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["isCurrent"]) for m in body] == [ + (f"{newer['urn']}#1", True), + (f"{older['urn']}#1", False), + ] + + +def test_measurements_sort_newest_published_first(client, session, data_provider, data_files, setup_router_db): + """Between two otherwise-equal current, unclassified measurements the newest-published sorts first — + published date outranks the urn tiebreak (here the newer set has the later urn, yet still leads).""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + first_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == first["urn"])) + second_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == second["urn"])) + first_db.published_date = date(2024, 1, 1) + second_db.published_date = date(2024, 6, 1) # later-published, and later urn + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + assert [m["variantUrn"] for m in response.json()] == [f"{second['urn']}#1", f"{first['urn']}#1"] + + +def test_measurements_tiebreak_on_urn(client, session, data_provider, data_files, setup_router_db): + """With every other sort key equal (both current, direct, unclassified, same published date) the urn is + the final, stable tiebreak — ascending.""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + for urn in (first["urn"], second["urn"]): + score_set = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == urn)) + score_set.published_date = date(2024, 1, 1) + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + returned = [m["variantUrn"] for m in response.json()] + assert returned == sorted([f"{first['urn']}#1", f"{second['urn']}#1"]) + + +def test_anonymous_cannot_see_private_measurement( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_cdna_measurement(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + # A content-addressed ClinGen id is public, but a private score set's measurement is filtered out. + assert response.status_code == 200 + assert response.json() == [] diff --git a/tests/routers/test_collections.py b/tests/routers/test_collections.py index b3df63538..899a30fba 100644 --- a/tests/routers/test_collections.py +++ b/tests/routers/test_collections.py @@ -1116,3 +1116,33 @@ def test_only_users_who_can_add_users_see_the_full_roster( assert orcids["admins"] == [TEST_USER["username"]] assert orcids["editors"] == [] assert orcids["viewers"] == [] + + +@pytest.mark.parametrize("route", ["add", "create", "update"]) +def test_cannot_add_unreadable_score_set_to_own_collection( + session, client, setup_router_db, extra_user_app_overrides, route +): + """Adding another user's private score set to your own collection reports it as missing, so membership + can neither probe for private records nor attach them.""" + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + if route == "create": + response = client.post( + "/api/v1/collections/", json={**TEST_COLLECTION, "score_set_urns": [private_score_set["urn"]]} + ) + else: + collection = create_collection(client) + if route == "add": + response = client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + else: + response = client.patch( + f"/api/v1/collections/{collection['urn']}", json={"score_set_urns": [private_score_set["urn"]]} + ) + + assert response.status_code == 404 + assert session.query(CollectionScoreSetAssociation).count() == 0 diff --git a/tests/routers/test_diseases.py b/tests/routers/test_diseases.py new file mode 100644 index 000000000..19c3c2bf9 --- /dev/null +++ b/tests/routers/test_diseases.py @@ -0,0 +1,51 @@ +"""Tests for the /diseases/search MONDO typeahead endpoint.""" + +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") +pytest.importorskip("cdot") +pytest.importorskip("fastapi") + +from mavedb.lib.exceptions import MondoServiceError + + +@pytest.fixture +def mock_mondo_search(monkeypatch): + async def fake_search(query, limit=20): + return [ + { + "code": "MONDO:0015263", + "label": "Brugada syndrome", + "iri": "https://purl.obolibrary.org/obo/MONDO_0015263", + } + ] + + monkeypatch.setattr("mavedb.routers.diseases.search_mondo", fake_search) + + +def test_search_diseases_returns_mappable_concepts(client, mock_mondo_search): + response = client.get("/api/v1/diseases/search", params={"q": "brugada"}) + + assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert data[0]["conceptType"] == "Disease" + assert data[0]["name"] == "Brugada syndrome" + assert data[0]["primaryCoding"]["code"] == "MONDO:0015263" + + +def test_search_diseases_requires_a_query(client): + assert client.get("/api/v1/diseases/search").status_code == 422 + + +def test_search_diseases_surfaces_ontology_outage(client, monkeypatch): + async def failing_search(query, limit=20): + raise MondoServiceError("Disease ontology service temporarily unavailable") + + monkeypatch.setattr("mavedb.routers.diseases.search_mondo", failing_search) + + response = client.get("/api/v1/diseases/search", params={"q": "brugada"}) + assert response.status_code == 503 + assert "Disease ontology service temporarily unavailable" in response.json()["message"] diff --git a/tests/routers/test_experiments.py b/tests/routers/test_experiments.py index e4f56785e..71c6729bd 100644 --- a/tests/routers/test_experiments.py +++ b/tests/routers/test_experiments.py @@ -2292,3 +2292,25 @@ def test_cannot_add_experiment_to_others_public_experiment_set( assert response.status_code == 403 response_data = response.json() assert f"insufficient permissions on experiment set with URN '{experiment_set_urn}'" in response_data["detail"] + + +def test_experiment_list_omits_unreadable_score_sets( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides +): + """A published experiment often holds a private score set, such as a new submission or an unpublished + successor; listing experiments must not name it to callers who can't read it.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + private_score_set = create_seq_score_set(client, published["experiment"]["urn"]) + + owner_list = client.get("/api/v1/experiments/").json() + assert [sorted(e["scoreSetUrns"]) for e in owner_list] == [sorted([published["urn"], private_score_set["urn"]])] + + with DependencyOverrider(anonymous_app_overrides): + anonymous_list = client.get("/api/v1/experiments/").json() + + assert [e["scoreSetUrns"] for e in anonymous_list] == [[published["urn"]]] + assert [e["numScoreSets"] for e in anonymous_list] == [1] diff --git a/tests/routers/test_hgvs.py b/tests/routers/test_hgvs.py index 6011953f2..b8a689dfb 100644 --- a/tests/routers/test_hgvs.py +++ b/tests/routers/test_hgvs.py @@ -24,6 +24,7 @@ INVALID_TRANSCRIPT = "NX_99999.1" VALID_VARIANT = VALID_NT_ACCESSION + ":c.1G>A" INVALID_VARIANT = VALID_NT_ACCESSION + ":c.1delA" +UNPARSEABLE_VARIANT = "not a valid hgvs string" HAS_PROTEIN_ACCESSION = "NM_000014.4" PROTEIN_ACCESSION = "NP_000005.2" @@ -64,6 +65,21 @@ def test_hgvs_validate_invalid(client, setup_router_db): assert "does not agree" in response.json()["detail"] +def test_hgvs_validate_unparseable(client, setup_router_db): + # A syntactically invalid HGVS string fails at the parse stage, before validation. This is a caller + # error and must surface as a 400 rather than escaping to the catch-all 500 handler. + payload = {"variant": UNPARSEABLE_VARIANT} + response = client.post("/api/v1/hgvs/validate", json=payload) + + assert response.status_code == 400 + + +def test_hgvs_validate_missing_field(client, setup_router_db): + response = client.post("/api/v1/hgvs/validate", json={}) + + assert response.status_code == 422 + + def test_hgvs_list_assemblies(client, setup_router_db): response = client.get("/api/v1/hgvs/assemblies") assert response.status_code == 200 diff --git a/tests/routers/test_mapped_variant.py b/tests/routers/test_mapped_variant.py new file mode 100644 index 000000000..5a2a33bfd --- /dev/null +++ b/tests/routers/test_mapped_variant.py @@ -0,0 +1,74 @@ +# ruff: noqa: E402 +"""Router tests for the retired ``/mapped-variants`` surface. + +The router itself is gone (see ``tests/routers/test_variant_annotation.py``); what remains at this +prefix is a set of 301 redirects onto the routes' new home under ``/variants``, kept so external +callers hitting the old public paths get pointed at the replacement instead of a bare 404. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote, quote_plus + + +TEST_URN = "urn:mavedb:00000001-a-1#1" + + +def test_redirect_mapped_variant(client): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}" + + +def test_redirect_carries_rfc8594_deprecation_headers(client): + """A client that follows the 301 silently still learns the resource is deprecated, and where it went.""" + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}", follow_redirects=False) + + assert response.headers["Deprecation"] == "true" + assert 'rel="successor-version"' in response.headers["Link"] + assert f"/api/v1/variants/{quote_plus(TEST_URN)}" in response.headers["Link"] + assert response.headers["Warning"].startswith("299 - ") + # Sunset stays unset until a removal date is ratified (deprecation.DEFAULT_SUNSET is None). + assert "Sunset" not in response.headers + + +@pytest.mark.parametrize( + "suffix", + ["va/study-result", "va/functional-statement", "va/pathogenicity-statement"], +) +def test_redirect_mapped_variant_va_routes(client, suffix): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}/{suffix}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}/{suffix}" + + +def test_redirect_carries_a_query_string_alongside_a_fragment_shaped_urn(client): + """TEST_URN's '#' would open a fragment if the target were built from request.url instead of the + ASGI scope, and a fragment swallows everything after it -- including a real query string.""" + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}?start=0&limit=1", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}?start=0&limit=1" + + +def test_redirect_mapped_variants_by_identifier(client): + identifier = "ga4gh:VA.0123456789abcdefghijklmnopqrstuv" + response = client.get( + f"/api/v1/mapped-variants/vrs/{identifier}?only_current=false", + follow_redirects=False, + ) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/vrs/{quote(identifier, safe='')}?only_current=false" + + +def test_redirect_mapped_variants_by_identifier_rejects_malformed_identifier(client): + response = client.get("/api/v1/mapped-variants/vrs/not-a-valid-identifier", follow_redirects=False) + + assert response.status_code == 422 diff --git a/tests/routers/test_mapped_variants.py b/tests/routers/test_mapped_variants.py deleted file mode 100644 index 6372faf78..000000000 --- a/tests/routers/test_mapped_variants.py +++ /dev/null @@ -1,630 +0,0 @@ -# ruff: noqa: E402 - -import json -from unittest.mock import patch - -import pytest - -from tests.helpers.util.user import change_ownership - -arq = pytest.importorskip("arq") -cdot = pytest.importorskip("cdot") -fastapi = pytest.importorskip("fastapi") - -from urllib.parse import quote_plus - -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import select -from sqlalchemy.orm.session import make_transient - -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet as ScoreSetDbModel -from mavedb.models.variant import Variant -from mavedb.view_models.mapped_variant import SavedMappedVariant -from tests.helpers.constants import ( - TEST_BIORXIV_IDENTIFIER, - TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, - TEST_PUBMED_IDENTIFIER, -) -from tests.helpers.util.common import deepcamelize -from tests.helpers.util.experiment import create_experiment -from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration -from tests.helpers.util.score_set import ( - create_seq_score_set_with_mapped_variants, - create_seq_score_set_with_variants, - publish_score_set, -) - - -def test_show_mapped_variant(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["id"] == 1 - - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_when_multiple_exist(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_when_none_exists(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_show_mapped_variant_study_result(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." - - ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_study_result_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_study_result_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_cannot_show_mapped_variant_study_result_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No study result exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_functional_impact_statement( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): - score_set = publish_score_set(client, score_set["urn"]) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." - - Statement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_functional_impact_statement_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): - score_set = publish_score_set(client, score_set["urn"]) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_insufficient_functional_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - # insufficient evidence = no (primary) calibrations - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1. Variant does not have sufficient evidence to evaluate its functional impact" - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_clinical_evidence_line( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): - score_set = publish_score_set(client, score_set["urn"]) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 200 - assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] - - VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_clinical_evidence_line_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): - score_set = publish_score_set(client, score_set["urn"]) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_insufficient_pathogenicity_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1; Variant does not have sufficient evidence to evaluate its pathogenicity" - in response_data["detail"] - ) - - -def test_show_mapped_variants_by_ga4gh_identifier(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_when_none_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - fake_mapped_variant_id = mapped_variant.pre_mapped["id"] - fake_mapped_variant_id = fake_mapped_variant_id[:-3] + "aaa" # Modify the ID to ensure it doesn't exist - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(fake_mapped_variant_id)}") - assert response.status_code == 404 - - -def test_show_mapped_variants_by_ga4gh_identifier_with_non_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - # Set the mapped variant to non-current - mapped_variant.current = False - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=false" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_show_mapped_variants_by_ga4gh_identifier_with_only_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - mapped_variant2 = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#2')) - assert mapped_variant is not None - assert mapped_variant2 is not None - - # Set the mapped variant to non-current - mapped_variant2.current = False - mapped_variant2.pre_mapped = mapped_variant.pre_mapped # Ensure both pre mapped blobs match besides current status - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=true" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - 1 - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_without_score_set_permissions( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - change_ownership(session, score_set["urn"], ScoreSetDbModel) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - assert response.status_code == 404 diff --git a/tests/routers/test_permissions.py b/tests/routers/test_permissions.py index b60a924e8..27a07b9e3 100644 --- a/tests/routers/test_permissions.py +++ b/tests/routers/test_permissions.py @@ -58,7 +58,7 @@ def test_contributor_gets_true_permission_from_others_experiment_set_add_experim assert response.json() -def test_get_false_permission_from_others_experiment_set_add_experiment_check(session, client, setup_router_db): +def test_get_not_found_from_others_experiment_set_add_experiment_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) change_ownership(session, experiment["experimentSetUrn"], ExperimentSetDbModel) @@ -67,8 +67,8 @@ def test_get_false_permission_from_others_experiment_set_add_experiment_check(se f"/api/v1/permissions/user-is-permitted/experiment-set/{experiment['experimentSetUrn']}/add_experiment" ) - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_cannot_get_permission_with_wrong_action_in_experiment_set(client, setup_router_db): @@ -167,14 +167,14 @@ def test_contributor_gets_true_permission_from_others_private_experiment_add_sco assert response.json() -def test_get_false_permission_from_others_private_experiment_add_score_set_check(session, client, setup_router_db): +def test_get_not_found_from_others_private_experiment_add_score_set_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/experiment/{experiment['urn']}/add_score_set") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_get_true_permission_from_others_public_experiment_add_score_set_check( @@ -198,24 +198,24 @@ def test_get_true_permission_from_others_public_experiment_add_score_set_check( assert response.json() -def test_get_false_permission_from_others_experiment_update_check(session, client, setup_router_db): +def test_get_not_found_from_others_experiment_update_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/experiment/{experiment['urn']}/update") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 -def test_get_false_permission_from_other_users_experiment_delete_check(session, client, setup_router_db): +def test_get_not_found_from_other_users_experiment_delete_check(session, client, setup_router_db): experiment = create_experiment(client) change_ownership(session, experiment["urn"], ExperimentDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/experiment/{experiment['urn']}/delete") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_cannot_get_permission_with_wrong_action_in_experiment(client, setup_router_db): @@ -318,37 +318,37 @@ def test_contributor_gets_false_permission_from_others_score_set_publish_check(s assert not response.json() -def test_get_false_permission_from_others_score_set_delete_check(session, client, setup_router_db): +def test_get_not_found_from_others_score_set_delete_check(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) change_ownership(session, score_set["urn"], ScoreSetDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/score-set/{score_set['urn']}/delete") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 -def test_get_false_permission_from_others_score_set_update_check(session, client, setup_router_db): +def test_get_not_found_from_others_score_set_update_check(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) change_ownership(session, score_set["urn"], ScoreSetDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/score-set/{score_set['urn']}/update") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 -def test_get_false_permission_from_others_score_set_publish_check(session, client, setup_router_db): +def test_get_not_found_from_others_score_set_publish_check(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) change_ownership(session, score_set["urn"], ScoreSetDbModel) response = client.get(f"/api/v1/permissions/user-is-permitted/score-set/{score_set['urn']}/publish") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_cannot_get_permission_with_wrong_action_in_score_set(client, setup_router_db): @@ -464,7 +464,7 @@ def test_get_true_permission_as_score_set_owner_on_others_investigator_provided_ assert response.json() -def test_get_false_permission_as_score_set_owner_on_others_community_score_calibration_update_check( +def test_get_not_found_as_score_set_owner_on_others_community_score_calibration_update_check( session, client, setup_router_db, admin_app_overrides ): experiment = create_experiment(client) @@ -477,8 +477,8 @@ def test_get_false_permission_as_score_set_owner_on_others_community_score_calib response = client.get(f"/api/v1/permissions/user-is-permitted/score-calibration/{score_calibration['urn']}/update") - assert response.status_code == 200 - assert not response.json() + # Unreadable resources are reported as missing, so the check can't confirm they exist. + assert response.status_code == 404 def test_get_false_permission_from_others_score_calibration_delete_check(session, client, setup_router_db): diff --git a/tests/routers/test_query_scaling.py b/tests/routers/test_query_scaling.py new file mode 100644 index 000000000..3715ffca3 --- /dev/null +++ b/tests/routers/test_query_scaling.py @@ -0,0 +1,257 @@ +# ruff: noqa: E402 +"""Query-count canary for every GET route with path parameters. + +Each route is called against two seeded score sets, one small and one four times larger, and must run no +more queries against the larger. A route whose count grows with its data is an N+1, which is invisible at +fixture sizes until it reaches a score set in production. + +Routes are found in the OpenAPI schema, so a new route is covered without a test of its own. Path +parameters are resolved by ``(resource segment, parameter)``: the segment before the parameter in the +path, since ``{urn}`` alone names a dozen different resources. A route under a resource the map doesn't +know fails, naming the pair to add to ``_RESOLVERS``, or a reason to add to ``_EXEMPT``. +""" + +import base64 +import hashlib +import re +from dataclasses import dataclass +from unittest.mock import patch +from urllib.parse import quote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.server_main import app +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_PUBMED_IDENTIFIER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.query_plan import captured_statements +from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration +from tests.helpers.util.score_set import create_seq_score_set, publish_score_set +from tests.helpers.util.variant import mock_worker_variant_insertion + +# Resource segments whose routes don't read the score set's variant data, or reach a service the test +# environment doesn't run. Each needs a reason. +_EXEMPT = { + "hgvs": "resolves through cdot, an external service", + "refget": "reads SeqRepo, not the variant data", + "seqrepo": "reads SeqRepo, not the variant data", + "orcid": "calls ORCID", + "publication-identifiers": "calls the publication registries", + "genes": "reads the gene normalizer, not the variant data", + "taxonomies": "reference data, not variant data", + "licenses": "reference data, not variant data", + "target-genes": "one target gene; not variant data", + "controlled-keywords": "reference data, not variant data", + "users": "one user; not variant data", + "collections": "one collection's metadata; not variant data", + "statistics": "site-wide counts, tracked in api#900", + "user-is-permitted": "one permission check; not variant data", + "job-runs": "one job run; not variant data", + "pipelines": "one pipeline; not variant data", +} + + +@dataclass(frozen=True) +class _World: + """The identifiers of one seeded score set and the records hanging off it.""" + + experiment_set_urn: str + experiment_urn: str + score_set_urn: str + variant_urn: str + measured_digest: str + protein_clingen_allele_id: str + calibration_urn: str + classification_id: int + + +_RESOLVERS = { + ("experiment-sets", "urn"): lambda world: world.experiment_set_urn, + ("experiments", "urn"): lambda world: world.experiment_urn, + ("score-sets", "urn"): lambda world: world.score_set_urn, + ("score-set", "score_set_urn"): lambda world: world.score_set_urn, + ("variants", "urn"): lambda world: world.variant_urn, + ("mapped-variants", "urn"): lambda world: world.variant_urn, + ("vrs", "identifier"): lambda world: world.measured_digest, + ("alleles", "identifier"): lambda world: world.measured_digest, + # The protein consequence every variant in the world shares, so its measurements grow with the world. + ("clingen-alleles", "clingen_allele_id"): lambda world: world.protein_clingen_allele_id, + ("score-calibrations", "urn"): lambda world: world.calibration_urn, + ("functional-classifications", "classification_id"): lambda world: world.classification_id, +} + +# Routes that answer every request with a fixed status by design, so there is nothing to scale. +_RETIRED = { + "/api/v1/score-sets/{urn}/mapped-variants", +} + +_PARAMETER = re.compile(r"{([^}]+)}") + +_PUBLICATION_FETCH = [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +def _parameterized_get_routes() -> list[str]: + return sorted( + path + for path, operations in app.openapi()["paths"].items() + if "get" in operations and _PARAMETER.search(path) and path not in _RETIRED + ) + + +def _resolve(path: str, world: _World) -> str: + """The concrete URL for ``path`` in ``world``. Raises ``KeyError`` naming the unresolved pair.""" + segments = path.split("/") + + def substitute(match: re.Match) -> str: + parameter = match.group(1) + index = segments.index(match.group(0)) + resolver = _RESOLVERS[(segments[index - 1], parameter)] + return quote(str(resolver(world)), safe=":") + + return _PARAMETER.sub(substitute, path) + + +def _exemption(path: str) -> str | None: + return next((reason for segment, reason in _EXEMPT.items() if f"/{segment}/" in path), None) + + +def _digest(text: str) -> str: + """A well-formed GA4GH VRS identifier, unique to ``text``.""" + return "ga4gh:VA." + base64.urlsafe_b64encode(hashlib.sha512(text.encode()).digest()[:24]).decode() + + +def _seed_world(client, session, data_provider, scores_csv, label: str) -> _World: + """A published score set whose variants each carry a coding measurement with VEP, gnomAD and ClinVar + records, its genomic projection, a convergent encoding and a protein consequence they all share, under a + published primary calibration that classifies every variant.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + mock_worker_variant_insertion(client, session, data_provider, score_set, scores_csv) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + calibration = create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + variants = session.scalars( + select(VariantDbModel) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).all() + vrs = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + for index, variant in enumerate(variants): + measured, convergent = GroupSpec(), GroupSpec() + seed_mapping_record( + session, + variant, + hgvs_assay_level=f"NM_000546.6:c.{1000 + index}G>A", + alleles=[ + AlleleSpec( + digest=_digest(f"{label}-measured-{index}"), + is_authoritative=True, + clingen_allele_id=f"CA{label}{index}", + group=measured, + vep_consequence="missense_variant", + clinvar_control_ids=(1,), + gnomad_variant_ids=(1,), + post_mapped=vrs, + ), + AlleleSpec( + digest=_digest(f"{label}-genomic-{index}"), level="genomic", group=measured, post_mapped=vrs + ), + AlleleSpec(digest=_digest(f"{label}-convergent-{index}"), group=convergent, post_mapped=vrs), + AlleleSpec( + digest=_digest(f"{label}-protein"), level="protein", clingen_allele_id=f"PA{label}", post_mapped=vrs + ), + ], + ) + + return _World( + experiment_set_urn=experiment["experimentSetUrn"], + experiment_urn=experiment["urn"], + score_set_urn=score_set["urn"], + variant_urn=variants[0].urn, + measured_digest=_digest(f"{label}-measured-0"), + protein_clingen_allele_id=f"PA{label}", + calibration_urn=calibration["urn"], + classification_id=calibration["functionalClassifications"][0]["id"], + ) + + +def _scores_csv(tmp_path, count: int): + """A scores file of ``count`` single-base substitutions against the minimal target (``ACGTTT``), scored + in the calibration's abnormal range so each variant carries pathogenicity evidence.""" + reference = "ACGTTT" + substitutions = [(position, base, alt) for position, base in enumerate(reference) for alt in "ACGT" if alt != base][ + :count + ] + rows = [ + f"n.{position + 1}{base}>{alt},{-2.0 - index / 10}" for index, (position, base, alt) in enumerate(substitutions) + ] + path = tmp_path / f"scores_{count}.csv" + path.write_text("hgvs_nt,score\n" + "\n".join(rows) + "\n") + return path + + +@pytest.fixture +def worlds(client, session, data_provider, setup_router_db, mock_publication_fetch, tmp_path): + small = _seed_world(client, session, data_provider, _scores_csv(tmp_path, 3), "small") + large = _seed_world(client, session, data_provider, _scores_csv(tmp_path, 12), "large") + return small, large + + +def _query_count(client, session, url: str) -> tuple[int, int]: + with captured_statements(session) as statements: + response = client.get(url) + return response.status_code, len(statements) + + +def test_every_parameterized_get_route_is_resolved_or_exempt(): + """A new route under an unknown resource needs a resolver or an exemption reason, not a silent skip.""" + unresolved = set() + for path in _parameterized_get_routes(): + if _exemption(path) is not None: + continue + segments = path.split("/") + for match in _PARAMETER.finditer(path): + key = (segments[segments.index(match.group(0)) - 1], match.group(1)) + if key not in _RESOLVERS: + unresolved.add((path, key)) + + assert not unresolved, f"Add a resolver to _RESOLVERS or a reason to _EXEMPT for: {sorted(unresolved)}" + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=True) +def test_no_route_runs_more_queries_on_a_larger_score_set(client, session, worlds): + small, large = worlds + failures = [] + for path in _parameterized_get_routes(): + if _exemption(path) is not None: + continue + + small_status, small_count = _query_count(client, session, _resolve(path, small)) + large_status, large_count = _query_count(client, session, _resolve(path, large)) + if small_status >= 400 or large_status >= 400: + failures.append(f"{path}: status {small_status}/{large_status}, so its query count isn't comparable") + elif large_count > small_count: + failures.append(f"{path}: {small_count} queries on the small score set, {large_count} on the large one") + + assert not failures, "\n".join(failures) diff --git a/tests/routers/test_refget.py b/tests/routers/test_refget.py index 760b9f022..9eb58f6f9 100644 --- a/tests/routers/test_refget.py +++ b/tests/routers/test_refget.py @@ -3,15 +3,36 @@ import pytest +pytestmark = pytest.mark.unit + arq = pytest.importorskip("arq") cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") biocommons = pytest.importorskip("biocommons") bioutils = pytest.importorskip("bioutils") +from mavedb.routers import refget from tests.helpers.constants import TEST_SEQREPO_INITIAL_STATE, VALID_ENSEMBL_IDENTIFIER +@pytest.mark.parametrize( + "env_value,expected_data_version", + [(None, "unknown"), ("/some/path/seqrepo/20240101", "20240101")], +) +def test_service_info(client, monkeypatch, env_value, expected_data_version): + if env_value is None: + monkeypatch.delenv("HGVS_SEQREPO_DIR", raising=False) + else: + monkeypatch.setenv("HGVS_SEQREPO_DIR", env_value) + + resp = client.get("/api/v1/refget/sequence/service-info") + assert resp.status_code == 200 + data = resp.json() + assert data["name"] == "MaveDB API" + assert data["seqrepo_data_version"] == expected_data_version + assert data["refget"]["identifier_types"] == ["refseq", "ensembl"] + + @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) def test_get_metadata_success(client, entry): alias = list(entry.keys())[0] @@ -45,6 +66,15 @@ def test_get_metadata_not_found(client): assert "Sequence not found" in resp.text +# The shared SeqRepo holds the target sequences of unpublished score sets, so lookups must not match by +# wildcard or digest prefix. +@pytest.mark.parametrize("path", ["{alias}", "{alias}/metadata"]) +@pytest.mark.parametrize("alias", ["ga4gh:SQ.%25", "MD5:%25", "%25", "0123456789"]) +def test_get_sequence_does_not_match_wildcards_or_digest_prefixes(client, path, alias): + resp = client.get(f"/api/v1/refget/sequence/{path.format(alias=alias)}") + assert resp.status_code == 404 + + @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) def test_get_sequence_success(client, entry): alias = list(entry.keys())[0] @@ -76,6 +106,86 @@ def test_get_sequence_with_range_query(client, entry): assert resp.text == metadata["seq"][start:end] +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_start(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + start = 1 + resp = client.get(f"/api/v1/refget/sequence/{alias}", params={"start": start}) + assert resp.status_code == 200 + assert resp.text == metadata["seq"][start:] + + +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_end(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + end = 3 + resp = client.get(f"/api/v1/refget/sequence/{alias}", params={"end": end}) + assert resp.status_code == 200 + assert resp.text == metadata["seq"][:end] + + +def test_get_sequence_range_header_and_query_params_conflict(client): + resp = client.get( + f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", + params={"start": 1}, + headers={"Range": "bytes=1-2"}, + ) + assert resp.status_code == 400 + assert "Cannot use both start/end query parameters and Range header" in resp.text + + +def test_get_sequence_invalid_query_range_only_start_negative(client): + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": -1}) + assert resp.status_code == 400 + assert "Invalid coordinates" in resp.text + + +def test_get_sequence_invalid_query_range_only_start_too_large(client): + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": 7}) + assert resp.status_code == 400 + assert "Invalid coordinates" in resp.text + + +def test_get_sequence_only_end_too_large_clamps(client): + # Unlike start, an over-long lone end is clamped rather than rejected. + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"end": 10}) + assert resp.status_code == 200 + assert resp.text == "GGGG" + + +@pytest.mark.parametrize( + "params,headers", + [ + ({}, {}), + ({"start": 1, "end": 3}, {}), + ({"start": 1}, {}), + ({"end": 3}, {}), + ({}, {"Range": "bytes=1-3"}), + ], +) +def test_get_sequence_content_length_matches_body(client, params, headers): + resp = client.get( + f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", + params=params, + # Accept-Encoding: identity is required — GZipMiddleware strips Content-Length from compressed responses. + headers={**headers, "Accept-Encoding": "identity"}, + ) + assert resp.status_code in (200, 206) + assert resp.headers["Content-Length"] == str(len(resp.content)) + + +def test_get_sequence_range_header_reports_requested_span(client): + resp = client.get( + f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", + headers={"Range": "bytes=1-2", "Accept-Encoding": "identity"}, + ) + assert resp.status_code == 206 + assert resp.headers["Content-Range"] == "bytes 1-2/4" + assert resp.headers["Content-Length"] == "2" + + def test_get_sequence_not_found(client): resp = client.get("/api/v1/refget/sequence/notfound") assert resp.status_code == 404 @@ -93,8 +203,8 @@ def test_get_sequence_multiple_ids(client): def test_get_sequence_invalid_header_range_coords_start_larger_than_end(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers={"Range": "bytes=12-10"}) - assert resp.status_code == 501 - assert "Invalid coordinates" in resp.text + assert resp.status_code == 400 + assert "Invalid range header format" in resp.text def test_get_sequence_invalid_header_range_coords_start_too_large(client): @@ -113,8 +223,18 @@ def test_get_sequence_invalid_header_range_coords_end_too_large(client): def test_get_sequence_invalid_query_range_coords_start_larger_than_end(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": 12, "end": 10}) - assert resp.status_code == 501 + assert resp.status_code == 416 assert "Invalid coordinates" in resp.text + assert "Content-Range" in resp.headers + + +def test_get_sequence_query_range_start_larger_than_end_on_circular_sequence_is_not_implemented(client, monkeypatch): + monkeypatch.setattr(refget, "CIRCULAR_SEQUENCE_ACCESSIONS", frozenset({VALID_ENSEMBL_IDENTIFIER})) + + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", params={"start": 12, "end": 10}) + + assert resp.status_code == 501 + assert "circular chromosomes are not supported" in resp.text def test_get_sequence_invalid_query_range_coords_start_too_large(client): @@ -136,3 +256,16 @@ def test_get_sequence_range_header_invalid(client): resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers=headers) assert resp.status_code == 400 assert "Invalid range header format" in resp.text + + +def test_get_sequence_range_header_invalid_inverted_by_one(client): + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers={"Range": "bytes=5-4"}) + assert resp.status_code == 400 + assert "Invalid range header format" in resp.text + + +def test_get_sequence_range_header_single_byte(client): + # Boundary case adjacent to the inverted range above: first-byte-pos == last-byte-pos is valid. + resp = client.get(f"/api/v1/refget/sequence/{VALID_ENSEMBL_IDENTIFIER}", headers={"Range": "bytes=1-1"}) + assert resp.status_code == 206 + assert resp.text == "G" diff --git a/tests/routers/test_score_calibrations.py b/tests/routers/test_score_calibrations.py index 16b4b4c31..c142ba8e0 100644 --- a/tests/routers/test_score_calibrations.py +++ b/tests/routers/test_score_calibrations.py @@ -14,6 +14,7 @@ from arq import ArqRedis from sqlalchemy import select +from mavedb.models.mondo_term import MondoTerm as MondoTermDbModel from mavedb.models.score_calibration import ScoreCalibration as CalibrationDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from tests.helpers.constants import ( @@ -149,6 +150,146 @@ def test_creating_user_can_get_score_calibration_when_private( assert calibration_response["private"] is True +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_create_score_calibration_without_disease_defaults_to_generic( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + disease = client.get(f"/api/v1/score-calibrations/{calibration['urn']}").json()["disease"] + assert disease["conceptType"] == "Disease" + assert disease["primaryCoding"]["code"] == "MONDO:0000001" + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_create_score_calibration_with_disease_resolves_mondo_concept( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, monkeypatch +): + async def fake_fetch(code): + return { + "code": code, + "label": "Brugada syndrome", + "iri": f"https://purl.obolibrary.org/obo/{code.replace(':', '_')}", + } + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + payload = { + **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), + "disease": "MONDO:0015263", + } + calibration = create_test_score_calibration_in_score_set_via_client(client, score_set["urn"], payload) + + disease = client.get(f"/api/v1/score-calibrations/{calibration['urn']}").json()["disease"] + assert disease["primaryCoding"]["code"] == "MONDO:0015263" + assert disease["name"] == "Brugada syndrome" + + stored = session.query(MondoTermDbModel).filter(MondoTermDbModel.code == "MONDO:0015263").one() + assert stored.label == "Brugada syndrome" + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_create_score_calibration_with_unknown_disease( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, monkeypatch +): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + payload = { + **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), + "disease": "MONDO:9999999", + "scoreSetUrn": score_set["urn"], + } + + response = client.post("/api/v1/score-calibrations/", json=payload) + + assert response.status_code == 422 + assert response.json()["detail"][0]["loc"] == ["body", "disease"] + assert "MONDO:9999999" in response.json()["detail"][0]["msg"] + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_modify_score_calibration_with_unknown_disease( + client, setup_router_db, mock_publication_fetch, session, data_provider, data_files, monkeypatch +): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + payload = { + **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), + "disease": "MONDO:9999999", + "scoreSetUrn": score_set["urn"], + } + + response = client.put(f"/api/v1/score-calibrations/{calibration['urn']}", json=payload) + + assert response.status_code == 422 + assert response.json()["detail"][0]["loc"] == ["body", "disease"] + stored = client.get(f"/api/v1/score-calibrations/{calibration['urn']}").json() + assert stored["disease"]["primaryCoding"]["code"] == "MONDO:0000001" + + @pytest.mark.parametrize( "mock_publication_fetch", [ @@ -5239,3 +5380,50 @@ def test_user_sees_calibrations_across_multiple_score_sets( returned_urns = {c["urn"] for c in calibrations} assert cal_1["urn"] in returned_urns assert cal_2["urn"] in returned_urns + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, + ] + ], + indirect=["mock_publication_fetch"], +) +@pytest.mark.parametrize("action", ["promote-to-primary", "demote-from-primary"]) +def test_score_set_owner_cannot_rank_or_see_private_community_calibration( + client, + setup_router_db, + mock_publication_fetch, + session, + data_provider, + data_files, + extra_user_app_overrides, + action, +): + """CHANGE_RANK admits the score set owner to community calibrations, but a private one is not theirs to + read, so the rank routes must not return or describe it.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, + session, + data_provider, + experiment["urn"], + data_files / "scores.csv", + ) + + with patch.object(ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + assert calibration["investigatorProvided"] is False + + response = client.post(f"/api/v1/score-calibrations/{calibration['urn']}/{action}") + + assert response.status_code == 404 + assert "functionalClassifications" not in response.json() diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 9e09743ea..fae869f33 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4,8 +4,10 @@ import json import re from copy import deepcopy -from datetime import date +from datetime import date, datetime, timezone from io import StringIO +import threading +from types import SimpleNamespace from unittest.mock import patch import jsonschema @@ -18,18 +20,27 @@ fastapi = pytest.importorskip("fastapi") from mavedb.lib.annotation.annotate import variant_study_result +from mavedb.lib.csv import build_limiter from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.exceptions import NonexistentOrcidUserError +from mavedb.lib.score_distribution import summarize_scores from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.collection import Collection as CollectionDbModel +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.enums.target_category import TargetCategory from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord from mavedb.models.pipeline import Pipeline -from mavedb.models.score_calibration import ScoreCalibration as ScoreCalibrationDbModel -from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel +from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel +from mavedb.view_models.lean_variant import LeanVariant +from mavedb.models.score_calibration import ScoreCalibration as ScoreCalibrationDbModel from mavedb.routers.score_sets import _annotation_stream_record from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate @@ -64,10 +75,13 @@ TEST_SAVED_GNOMAD_VARIANT, TEST_SAVED_TAXONOMY, TEST_USER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, VALID_CLINGEN_CA_ID, + VALID_VARIANT_URN, ) from tests.helpers.dependency_overrider import DependencyOverrider -from tests.helpers.mocks.factories import create_mock_mapped_variant +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record from tests.helpers.util.common import ( create_failing_side_effect, deepcamelize, @@ -85,15 +99,15 @@ create_seq_score_set, create_seq_score_set_with_mapped_variants, create_seq_score_set_with_variants, - link_clinical_controls_to_mapped_variants, - link_clinvar_control_to_mapped_variant, - link_gnomad_variants_to_mapped_variants, + link_clinical_controls_to_alleles, publish_score_set, + seed_annotation_substrate, + seed_csv_substrate, + shift_hgvs_position, ) +from tests.helpers.util.collection import create_collection from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( - clear_first_mapped_variant_post_mapped, - create_mapped_variants_for_score_set, mock_worker_variant_insertion, ) @@ -203,6 +217,37 @@ def test_create_score_set_with_contributor(client, mock_publication_fetch, setup assert response.status_code == 200 +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_create_score_set_with_score_calibration_with_unknown_disease( + client, mock_publication_fetch, setup_router_db, monkeypatch +): + async def fake_fetch(code): + return None + + monkeypatch.setattr("mavedb.lib.mondo_ols.fetch_mondo_term", fake_fetch) + + experiment = create_experiment(client) + score_set = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set["experimentUrn"] = experiment["urn"] + score_set["scoreCalibrations"] = [ + {**deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED), "disease": "MONDO:9999999"} + ] + + response = client.post("/api/v1/score-sets/", json=score_set) + + assert response.status_code == 422 + assert response.json()["detail"][0]["loc"] == ["body", "scoreCalibrations", 0, "disease"] + + @pytest.mark.parametrize( "mock_publication_fetch", [ @@ -286,6 +331,33 @@ def test_cannot_create_score_set_with_class_based_calibration(client, mock_publi assert "Class-based calibrations are not supported on score set creation" in response_data["detail"] +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + ( + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ) + ], + indirect=["mock_publication_fetch"], +) +def test_cannot_create_score_set_with_calibration_controls(client, mock_publication_fetch, setup_router_db): + experiment = create_experiment(client) + score_set = deepcopy(TEST_MINIMAL_SEQ_SCORESET) + score_set["experimentUrn"] = experiment["urn"] + calibration = deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + # Controls reference variants that don't exist until the scores file is processed after creation. + calibration["controls"] = [{"variantUrn": VALID_VARIANT_URN, "clinicalStatus": "pathogenic"}] + score_set.update({"scoreCalibrations": [calibration]}) + + response = client.post("/api/v1/score-sets/", json=score_set) + assert response.status_code == 409 + response_data = response.json() + assert "Calibration controls are not supported on score set creation" in response_data["detail"] + + @pytest.mark.parametrize( "mock_publication_fetch", [ @@ -821,6 +893,22 @@ def test_get_own_private_score_set(client, setup_router_db): assert (key, expected_response[key]) == (key, response_data[key]) +def test_get_score_set_serves_stored_score_distribution(session, client, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + assert "scoreDistribution" not in client.get(f"/api/v1/score-sets/{score_set['urn']}").json() + + item = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"])).one() + item.score_distribution = summarize_scores([-1.0, 0.5, 2.0, None]) + session.commit() + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}") + assert response.status_code == 200 + distribution = response.json()["scoreDistribution"] + assert (distribution["min"], distribution["max"], distribution["nullCount"]) == (-1.0, 2.0, 1) + assert sum(distribution["counts"]) == 3 + + def test_cannot_get_other_user_private_score_set(session, client, setup_router_db): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) @@ -1200,6 +1288,73 @@ def test_add_score_set_variants_scores_only_endpoint(client, setup_router_db, da assert score_set == response_data +def test_add_score_set_variants_is_refused_on_a_published_score_set( + session, data_provider, client, setup_router_db, data_files, mock_s3_client +): + """Publishing freezes a score set's scores; the upload endpoint must refuse them afterwards. + + The UI only offers score editing while a score set is private, but that is not a guarantee: the + endpoint is reachable directly, and a re-upload would replace variants other records already + point at. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + scores_csv_path = data_files / "scores.csv" + with ( + open(scores_csv_path, "rb") as scores_file, + patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as queue, + patch.object(mock_s3_client, "upload_fileobj", return_value=None), + ): + response = client.post( + f"/api/v1/score-sets/{published['urn']}/variants/data", + files={"scores_file": (scores_csv_path.name, scores_file, "text/csv")}, + ) + # The refusal must land before any work is queued or uploaded. + queue.assert_not_called() + + assert response.status_code == 403 + + +def test_patch_published_score_set_with_scores_file_is_refused_without_applying_the_update( + session, data_provider, client, setup_router_db, data_files, mock_s3_client +): + """A refused score upload must not leave the request's metadata half committed. + + The combined endpoint commits metadata before it reaches the enqueue step, so the SET_SCORES + check has to run before the update is applied rather than only before the job is queued. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + scores_csv_path = data_files / "scores.csv" + with ( + open(scores_csv_path, "rb") as scores_file, + patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as queue, + patch.object(mock_s3_client, "upload_fileobj", return_value=None), + ): + response = client.patch( + f"/api/v1/score-sets-with-variants/{published['urn']}", + data={"title": "Retitled after publication"}, + files={"scores_file": (scores_csv_path.name, scores_file, "text/csv")}, + ) + queue.assert_not_called() + + assert response.status_code == 403 + + # The title edit rode along with the refused upload, so it must not have been applied either. + refreshed = client.get(f"/api/v1/score-sets/{published['urn']}").json() + assert refreshed["title"] == published["title"] + + def test_add_score_set_variants_scores_and_counts_endpoint( session, client, setup_router_db, data_files, mock_s3_client ): @@ -1641,7 +1796,7 @@ def test_upload_score_set_variant_data_deletes_s3_files_when_pipeline_creation_f with ( open(scores_csv_path, "rb") as scores_file, - patch("mavedb.routers.score_sets.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), + patch("mavedb.lib.workflow.kickoff.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), patch.object(mock_s3_client, "upload_fileobj", return_value=None), ): response = client.post( @@ -1754,6 +1909,59 @@ def test_publish_score_set(session, data_provider, client, setup_router_db, data assert all([variant.urn.startswith("urn:mavedb:") for variant in score_set_variants]) +def test_cannot_publish_an_already_published_score_set(session, data_provider, client, setup_router_db, data_files): + """Publishing assigns a fresh URN unconditionally, so a second publish would rename a public record.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + response = client.post(f"/api/v1/score-sets/{published_score_set['urn']}/publish") + worker_queue.assert_not_called() + + assert response.status_code == 409 + assert "already been published" in response.json()["detail"] + + # The URN the caller already shared still resolves to this record. + unchanged = client.get(f"/api/v1/score-sets/{published_score_set['urn']}") + assert unchanged.status_code == 200 + assert unchanged.json()["urn"] == published_score_set["urn"] + + +def test_publish_score_set_returns_conflict_when_assigned_urn_is_taken( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set_1 = create_seq_score_set(client, experiment["urn"]) + score_set_1 = mock_worker_variant_insertion(client, session, data_provider, score_set_1, data_files / "scores.csv") + score_set_2 = create_seq_score_set(client, experiment["urn"]) + score_set_2 = mock_worker_variant_insertion(client, session, data_provider, score_set_2, data_files / "scores.csv") + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set_1 = publish_score_set(client, score_set_1["urn"]) + + # Simulates a concurrent publication that minted the same URN between this request's read and its write. + with ( + patch( + "mavedb.routers.score_sets.generate_score_set_urn", + return_value=published_score_set_1["urn"], + ), + patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue, + ): + response = client.post(f"/api/v1/score-sets/{score_set_2['urn']}/publish") + worker_queue.assert_not_called() + + assert response.status_code == 409 + assert "already in use" in response.json()["detail"] + + unpublished = client.get(f"/api/v1/score-sets/{score_set_2['urn']}") + assert unpublished.status_code == 200 + assert unpublished.json()["private"] is True + + def test_publish_score_set_discards_pipeline_when_entrypoint_enqueue_fails( session, data_provider, client, setup_router_db, data_files ): @@ -2152,7 +2360,10 @@ def test_multiple_score_set_meta_analysis_single_experiment( published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2192,7 +2403,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2234,7 +2448,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiments( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2290,7 +2507,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco published_score_set_1_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1_1['urn']}")).json() assert meta_score_set_1["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1_1["urn"], published_score_set_1_2["urn"]] + [ + published_score_set_1_1["urn"], + published_score_set_1_2["urn"], + ] ) assert published_score_set_1_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_1["urn"]] @@ -2307,7 +2527,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco ) published_score_set_2_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_2_1['urn']}")).json() assert meta_score_set_2["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_2_1["urn"], published_score_set_2_2["urn"]] + [ + published_score_set_2_1["urn"], + published_score_set_2_2["urn"], + ] ) assert published_score_set_2_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_2["urn"]] @@ -2429,7 +2652,10 @@ def test_multiple_score_set_meta_analysis_single_experiment_with_different_creat published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2471,7 +2697,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_with_differen published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -3350,6 +3579,38 @@ def test_can_delete_own_private_scoreset(session, data_provider, client, setup_r assert response.status_code == 200 +def test_can_delete_own_private_scoreset_with_mapped_variants( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + variant = session.scalars( + select(VariantDbModel).join(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set["urn"]) + ).first() + variant_ids = [variant.id] + session.add( + MappingRecord( + variant_id=variant.id, score_set_id=variant.score_set_id, assay_level="cdna", mapping_api_version="test.0.0" + ) + ) + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + ) + ) + session.commit() + + response = client.delete(f"/api/v1/score-sets/{score_set['urn']}") + + assert response.status_code == 200 + assert not session.scalars(select(MappingRecord).where(MappingRecord.variant_id.in_(variant_ids))).first() + assert not session.scalars(select(AnnotationEvent).where(AnnotationEvent.variant_id.in_(variant_ids))).first() + + def test_cannot_delete_own_published_scoreset(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) @@ -3723,7 +3984,17 @@ def test_download_variants_data_file( score_set = create_seq_score_set(client, experiment["urn"]) score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3750,17 +4021,23 @@ def test_download_variants_data_file( ] ) rows = list(reader) - for row in rows: + # seed_csv_substrate gives each variant its own allele, so each row's HGVS is the seeded one shifted by + # the row's position (the first row is unshifted). + for index, row in enumerate(rows): if has_hgvs_g: - assert row["mavedb.post_mapped_hgvs_g"] == mapped_variant["hgvs_g"] - assert row["mavedb.post_mapped_hgvs_c"] == mapped_variant["hgvs_c"] - assert row["mavedb.post_mapped_hgvs_p"] == mapped_variant["hgvs_p"] - assert row["mavedb.post_mapped_hgvs_at_assay_level"] == mapped_variant["hgvs_assay_level"] + assert row["mavedb.post_mapped_hgvs_g"] == shift_hgvs_position(mapped_variant["hgvs_g"], index) + assert row["mavedb.post_mapped_hgvs_c"] == shift_hgvs_position(mapped_variant["hgvs_c"], index) + assert row["mavedb.post_mapped_hgvs_p"] == shift_hgvs_position(mapped_variant["hgvs_p"], index) + assert row["mavedb.post_mapped_hgvs_at_assay_level"] == shift_hgvs_position( + mapped_variant["hgvs_assay_level"], index + ) elif has_hgvs_p: assert row["mavedb.post_mapped_hgvs_g"] == "NA" assert row["mavedb.post_mapped_hgvs_c"] == "NA" - assert row["mavedb.post_mapped_hgvs_p"] == mapped_variant["hgvs_p"] - assert row["mavedb.post_mapped_hgvs_at_assay_level"] == mapped_variant["hgvs_assay_level"] + assert row["mavedb.post_mapped_hgvs_p"] == shift_hgvs_position(mapped_variant["hgvs_p"], index) + assert row["mavedb.post_mapped_hgvs_at_assay_level"] == shift_hgvs_position( + mapped_variant["hgvs_assay_level"], index + ) else: assert row["mavedb.post_mapped_hgvs_g"] == "NA" assert row["mavedb.post_mapped_hgvs_c"] == "NA" @@ -3812,6 +4089,51 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d assert "hgvs_splice" not in columns +@pytest.mark.parametrize("path", ["scores", "counts", "variants/data"]) +def test_csv_routes_raise_the_statement_timeout(session, data_provider, client, setup_router_db, data_files, path): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion( + client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with patch("mavedb.routers.score_sets.allow_long_statements") as allow_long_statements: + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}") + + assert response.status_code == 200 + allow_long_statements.assert_called_once() + + +@pytest.mark.parametrize("path", ["scores", "counts", "variants/data"]) +def test_csv_routes_refuse_large_builds_when_every_slot_is_held( + session, data_provider, client, setup_router_db, data_files, monkeypatch, path +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion( + client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + held = threading.BoundedSemaphore(1) + held.acquire() + monkeypatch.setattr(build_limiter, "_slots", held) + monkeypatch.setattr(build_limiter, "CSV_BUILD_MIN_ROWS", 1) + monkeypatch.setattr(build_limiter, "CSV_BUILD_WAIT_SECONDS", 0.01) + + refused = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}") + + assert refused.status_code == 503 + assert refused.headers["retry-after"] == str(build_limiter.RETRY_AFTER_SECONDS) + # A limit under the threshold is never refused, which keeps the score set page's previews working. + monkeypatch.setattr(build_limiter, "CSV_BUILD_MIN_ROWS", 2) + preview = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}?limit=1") + assert preview.status_code == 200 + + # Deprecated query-parameter aliases. Galaxy and other external tooling call these endpoints, so the old # names keep working for a release rather than being silently ignored. def test_deprecated_drop_na_columns_still_drops_unused_hgvs_columns( @@ -3837,7 +4159,12 @@ def test_deprecated_include_post_mapped_hgvs_adds_the_mavedb_namespace( experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published_score_set = publish_score_set(client, score_set["urn"]) @@ -4015,7 +4342,16 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r download_scores_and_counts_csv = download_scores_and_counts_csv_response.text reader = csv.DictReader(StringIO(download_scores_and_counts_csv)) assert sorted(reader.fieldnames) == sorted( - ["accession", "hgvs_nt", "hgvs_pro", "scores.score", "scores.s_0", "scores.s_1", "counts.c_0", "counts.c_1"] + [ + "accession", + "hgvs_nt", + "hgvs_pro", + "scores.score", + "scores.s_0", + "scores.s_1", + "counts.c_0", + "counts.c_1", + ] ) @@ -4037,7 +4373,17 @@ def test_download_scores_counts_and_post_mapped_variants_file( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4075,8 +4421,14 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + # Seed mapped alleles with a VEP consequence on the authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4099,13 +4451,15 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants then set ClinGen allele id for first mapped variant - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles, with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4123,11 +4477,31 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie def test_download_gnomad_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) - # Link a gnomAD variant to the first mapped variant (version may not match export filter) - score_set = create_seq_score_set_with_mapped_variants( - client, session, data_provider, experiment["urn"], data_files / "scores.csv" + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # The CSV gnomAD column reflects the served release (v4.1); seed one and link it to the first variant's + # authoritative allele. + gnomad_variant = GnomADVariantDbModel( + db_name="gnomAD", + db_identifier="10-1-A-G", + db_version="v4.1", + allele_count=3, + allele_number=1613510, + allele_frequency=3 / 1613510, + faf95_max=6.8e-07, + faf95_max_ancestry="nfe", + ) + session.add(gnomad_variant) + session.commit() + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + gnomad_variant_ids=[gnomad_variant.id], + annotate="first", ) - link_gnomad_variants_to_mapped_variants(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4139,21 +4513,47 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames + rows = list(reader) + assert rows[0]["gnomad.gnomad_af"] == str(3 / 1613510) + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows[1:]) -def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( +def test_download_gnomad_file_serves_variants_linked_to_another_release( session, data_provider, client, setup_router_db, data_files ): - """A variant linked only to a gnomAD record of another version must still appear, with an NA frequency. + """A variant whose live gnomAD link is to an older release is served, labeled with that release. - The version filter belongs in the join's ON clause; in a WHERE it silently drops the variant row. + Two properties at once. The row must survive regardless of gnomAD linkage — the join's ON clause is + what preserves it, and the same predicate in a WHERE silently drops the variant. And the frequency + itself is read off the live link rather than filtered to the release the deployment currently serves: + ingestion only visits alleles a release covers, so alleles sitting at an older release are the steady + state, and filtering them out made their frequencies permanently NA. """ experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - # The seeded gnomAD variant's version deliberately differs from the configured export version. - link_gnomad_variants_to_mapped_variants(session, score_set) + # Deliberately not the release the deployment serves (v4.1): this is an allele a newer release did + # not cover, so its earlier link is still the live one. + gnomad_variant = GnomADVariantDbModel( + db_name="gnomAD", + db_identifier="10-1-A-G", + db_version="v2.1.1", + allele_count=3, + allele_number=1613510, + allele_frequency=3 / 1613510, + faf95_max=6.8e-07, + faf95_max_ancestry="nfe", + ) + session.add(gnomad_variant) + session.commit() + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + gnomad_variant_ids=[gnomad_variant.id], + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4164,7 +4564,8 @@ def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( rows = list(csv.DictReader(StringIO(response.text))) assert len(rows) == 3, "every variant must be present regardless of linked gnomAD versions" - assert all(row["gnomad.gnomad_af"] == "NA" for row in rows) + assert all(row["gnomad.gnomad_af"] == str(3 / 1613510) for row in rows) + assert all(row["gnomad.gnomad_version"] == "v2.1.1" for row in rows) def test_download_clingen_and_vep_file_in_variant_data_path( @@ -4175,13 +4576,16 @@ def test_download_clingen_and_vep_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a VEP consequence (all variants) and a ClinGen id (first variant only). + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4207,13 +4611,15 @@ def test_download_clingen_and_scores_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4245,10 +4651,18 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, # The ClinVar control seeded in setup_router_db has db_version="11_2024", mapping to namespace clinvar.2024_11. clinvar_namespace = "clinvar.2024_11" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + # ClinVar control id=1 (db_version 11_2024) links to only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4279,10 +4693,17 @@ def test_download_clinvar_namespace_with_no_matching_version( # clinvar.2023_01 does not match the seeded control (11_2024), so all rows should be NA. clinvar_namespace = "clinvar.2023_01" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4309,10 +4730,17 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( matching_ns = "clinvar.2024_11" # matches db_version="11_2024" seeded in setup_router_db non_matching_ns = "clinvar.2023_01" # no controls with this version experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -4340,13 +4768,98 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( assert all(row[f"{non_matching_ns}.clinical_review_status"] == "NA" for row in rows) -def test_invalid_clinvar_namespace_returns_422(client, setup_router_db, data_files): - """A clinvar namespace with an out-of-range month (13) is rejected with 422.""" +def test_csv_namespaces_as_of_matches_what_the_as_of_download_can_fill( + session, data_provider, client, setup_router_db, data_files +): + """Discovery and download must answer for the same instant. + + Pinned to "now", the picker offers mapping namespaces an `as_of` download returns entirely NA, and + omits ones it could have filled. The ClinVar case is sharpest: the live release set decides which + `clinvar.YYYY_MM` headers exist at all. + """ experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") - response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants/data?namespaces=clinvar.2024_13") - assert response.status_code == 422 + seeded_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + url = f"/api/v1/score-sets/{published['urn']}/csv-namespaces" + + current = client.get(url) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + assert "vep" in {entry["namespace"] for entry in current.json()} + + # Before the mapping was live: the mapping-derived namespaces are not offered, because the + # `as_of` download at that instant could not fill them. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(url, params={"as_of": past.isoformat()}) + assert before.status_code == 200 + assert before.headers["X-As-Of"] == past.isoformat() + assert "vep" not in {entry["namespace"] for entry in before.json()} + + +def test_download_variant_data_as_of_reconstructs_annotation_layer( + session, data_provider, client, setup_router_db, data_files +): + """`as_of` time-travels the CSV annotation layer: an instant before the annotation was live yields NA, + while the current view yields the value. The resolved instant is echoed in X-As-Of.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # Seed the whole substrate live as of 2020 — live "now" but not at an as_of in 2000. + seeded_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + base_url = f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + + # Current: the annotation is live, and X-As-Of echoes "current". + current = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false"}) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + current_rows = list(csv.DictReader(StringIO(current.text))) + assert any(row["vep.vep_functional_consequence"] == "missense_variant" for row in current_rows) + + # An instant before the annotation existed: nothing live -> every VEP value is NA, and X-As-Of echoes it. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false", "as_of": past.isoformat()}) + assert before.status_code == 200 + assert datetime.fromisoformat(before.headers["X-As-Of"]) == past + before_rows = list(csv.DictReader(StringIO(before.text))) + assert before_rows # variants (and their immutable scores) are still present + assert all(row["vep.vep_functional_consequence"] == "NA" for row in before_rows) + + +def test_invalid_clinvar_namespace_returns_422(client, setup_router_db, data_files): + """A clinvar namespace with an out-of-range month (13) is rejected with 422.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants/data?namespaces=clinvar.2024_13") + assert response.status_code == 422 def test_unrecognized_namespace_returns_422(client, setup_router_db, data_files): @@ -4368,7 +4881,7 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") assert response.status_code == 200 @@ -4376,15 +4889,66 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ response_data = response.json() assert len(response_data) == 2 for control in response_data: - mapped_variants = control.pop("mappedVariants") - assert len(mapped_variants) == 1 + clinvar_links = control.pop("clinvarLinks") + assert len(clinvar_links) == 1 + # Each link carries the digest of the allele the control annotates — the D78 precedence signal. + assert clinvar_links[0]["alleleDigest"] in ("clinical-control-allele-0", "clinical-control-allele-1") assert all( control[k] in (TEST_SAVED_CLINVAR_CONTROL[k], TEST_SAVED_GENERIC_CLINICAL_CONTROL[k]) for k in TEST_SAVED_CLINVAR_CONTROL.keys() - if k != "mappedVariants" + if k != "clinvarLinks" ) +def test_clinical_controls_tag_each_link_with_the_annotating_allele_digest( + client, setup_router_db, session, data_provider, data_files +): + """A protein-change variant whose DNA siblings carry conflicting ClinVar calls surfaces *both* + controls against the *same* variant URN — distinguishable only by the digest of the allele each + one annotates. That digest is D78's precedence signal: the control on the variant's authoritative + (assayed-level) allele is the direct call; the control on a projection sibling is the fallback. + Without it the two collapse to one variant and precedence is unimplementable.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = session.scalars( + select(VariantDbModel.urn) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).first() + + # One variant, two alleles in its equivalence class: the assayed (authoritative) allele carries the + # ClinVar control (id 1); a projection sibling carries the generic control (id 2). + seed_mapping_record( + session, + variant_urn, + assay_level="protein", + alleles=[ + AlleleSpec(digest="assayed-level-allele", level="protein", is_authoritative=True, clinvar_control_ids=[1]), + AlleleSpec( + digest="projection-sibling-allele", level="cdna", is_authoritative=False, clinvar_control_ids=[2] + ), + ], + ) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") + assert response.status_code == 200 + + controls = {c["dbIdentifier"]: c for c in response.json()} + assert len(controls) == 2 + + assayed = controls[TEST_SAVED_CLINVAR_CONTROL["dbIdentifier"]] # control 1, on the authoritative allele + sibling = controls[TEST_SAVED_GENERIC_CLINICAL_CONTROL["dbIdentifier"]] # control 2, on the sibling + + # Both controls point at the same variant, so only the annotating allele's digest tells them apart. + assert [link["variantUrn"] for link in assayed["clinvarLinks"]] == [variant_urn] + assert [link["variantUrn"] for link in sibling["clinvarLinks"]] == [variant_urn] + assert assayed["clinvarLinks"][0]["alleleDigest"] == "assayed-level-allele" + assert sibling["clinvarLinks"][0]["alleleDigest"] == "projection-sibling-allele" + + @pytest.mark.parametrize("clinical_control", [TEST_SAVED_CLINVAR_CONTROL, TEST_SAVED_GENERIC_CLINICAL_CONTROL]) @pytest.mark.parametrize( "parameters", [[("db", "dbName")], [("version", "dbVersion")], [("db", "dbName"), ("version", "dbVersion")]] @@ -4396,7 +4960,7 @@ def test_can_fetch_current_clinical_controls_for_score_set_with_parameters( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) query_string = "?" for param, accessor in parameters: @@ -4421,7 +4985,7 @@ def test_cannot_fetch_clinical_controls_for_nonexistent_score_set( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/clinical-controls") @@ -4456,7 +5020,7 @@ def test_can_fetch_current_clinical_control_options_for_score_set( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") assert response.status_code == 200 @@ -4481,23 +5045,176 @@ def test_clinical_control_options_exclude_non_current(client, setup_router_db, s score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) - # Mark all mapped variants as non-current to simulate stale mapping data. - mapped_variants = session.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - for mv in mapped_variants: - mv.current = False + # The options query only surfaces live allele → ClinVar links; retiring them (as a re-map would, + # by stamping valid_to) must leave no current controls and 404. Test-isolated DB, so these are + # the only links present. + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.valid_to.is_(None))).all() + for link in links: + link.valid_to = datetime.now(timezone.utc) session.commit() response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") assert response.status_code == 404 +######################################################################################################################## +# Downloading a score set's variant details (VRS + Cat-VRS + annotations) +######################################################################################################################## + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_get_score_set_variant_details_withholds_a_private_calibration( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + """Reading a score set does not entitle a caller to a private calibration's classifications. + + The whole-set export resolved calibration visibility by trusting the fetch helper to have narrowed + ``score_calibrations`` in place. That narrowing moved into the response builder, which this route does + not go through, so every calibration on the score set became visible to anyone who could read it -- + the single-variant twin filters correctly, and the two serve the same envelope. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set, pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X) + + # A community calibration on someone else's score set: private, and readable only by its creator. + calibration_item = session.query(ScoreCalibrationDbModel).filter_by(urn=calibration["urn"]).one() + calibration_item.investigator_provided = False + session.commit() + change_ownership(session, calibration["urn"], ScoreCalibrationDbModel) + + owner_records = parse_ndjson_response(client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details")) + assert owner_records, "the score set's own variants must still stream" + assert not any( + classification.get("calibrationId") == calibration_item.id + for record in owner_records + for classification in (record.get("classifications") or []) + ), "a calibration the caller cannot read must not contribute classifications" + + +def test_get_score_set_variant_details_returns_ndjson(client, session, data_provider, data_files, setup_router_db): + """The whole-set variant-detail export streams NDJSON — one VariantDetail per mapped variant, each + carrying the flat preMapped/postMapped VRS pair and the spec-pure Cat-VRS.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urns = { + variant.urn + for variant in seed_annotation_substrate(session, score_set, pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X) + } + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/x-ndjson") + assert response.headers["X-As-Of"] == "current" + assert response.headers["X-Total-Count"] == str(len(variant_urns)) + assert response.headers["X-Stream-Type"] == "variant-detail" + + records = parse_ndjson_response(response) + assert len(records) == len(variant_urns) + assert {record["urn"] for record in records} == variant_urns + for record in records: + # The flat VRS pair for VRS consumers, and the spec-pure Cat-VRS for the full picture. + assert record["preMapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert record["postMapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert record.get("molecularRepresentation") is not None + + +def test_get_score_set_variant_details_omits_unmapped_variants( + client, session, data_provider, data_files, setup_router_db +): + """Only mapped variants carry VRS/detail, so an un-mapped variant is omitted — the record count is + the mapped subset, not the whole score set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variants = seed_annotation_substrate(session, score_set, skip_first=True) + unmapped_urn = variants[0].urn # read before the request expires the instances + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + + records = parse_ndjson_response(response) + assert len(records) == score_set["numVariants"] - 1 + assert unmapped_urn not in {record["urn"] for record in records} + + +def test_get_score_set_variant_details_for_unmapped_returns_empty( + client, session, data_provider, data_files, setup_router_db +): + """A score set with no live mapping records has no annotatable variants — an empty NDJSON stream with + X-Total-Count 0, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # No new-substrate mapping records → no annotatable variants. + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +def test_cannot_get_variant_details_for_nonexistent_score_set(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variant-details") + assert response.status_code == 404 + + +def test_score_set_mapped_variants_is_permanently_removed(client): + """The old JSON-array ``mapped-variants`` route is gone in favor of the streaming NDJSON + ``variant-details`` route; it returns 410 rather than redirecting since the two are not + wire-compatible.""" + urn = "urn:mavedb:00000001-a-1" + response = client.get(f"/api/v1/score-sets/{urn}/mapped-variants") + + assert response.status_code == 410 + assert response.json()["detail"] == ( + f"GET /score-sets/{urn}/mapped-variants has been removed. Use GET /score-sets/{urn}/variant-details instead." + ) + + +def test_get_score_set_variant_details_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """``as_of`` time-travels the molecular layer: a far-future instant sees the freshly-seeded substrate + as live (the full set), a far-past instant sees no live mapping (an empty stream). Either way 200 — + ``as_of`` is a filter, so an empty match is an empty collection, never a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + url = f"/api/v1/score-sets/{score_set['urn']}/variant-details" + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] + + ######################################################################################################################## # Fetching annotated variants for a score set ######################################################################################################################## @@ -4516,9 +5233,14 @@ def test_cannot_get_annotated_variants_for_nonexistent_score_set(client, setup_r @pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) -def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( +def test_get_annotated_variants_for_score_set_with_no_mapped_variants_streams_empty( client, session, data_provider, data_files, setup_router_db, annotation_type ): + """A score set that exists but has no annotatable variants is an empty collection, not a 404. + + The route resolves (the URN names a real, readable score set), so it returns 200 with an empty NDJSON + body and ``X-Total-Count: 0``. 404 is reserved for an unresolvable URN or a permission failure. + """ experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -4543,13 +5265,47 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( assert "hgvs_splice" not in columns response = client.get(f"/api/v1/score-sets/{publish_score_set['urn']}/annotated-variants/{annotation_type}") - response_data = response.json() - assert response.status_code == 404 - assert ( - f"No mapped variants associated with score set URN {publish_score_set['urn']} were found" - in response_data["detail"] + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +@pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) +def test_annotated_variants_honor_as_of(client, session, data_provider, data_files, setup_router_db, annotation_type): + """The streaming annotation routes time-travel their annotatable set via ``as_of``. + + ``as_of`` is threaded into ``get_annotatable_variants`` (which allele links are live at that instant), + not only into the per-variant context. So a far-future instant sees the freshly-seeded substrate as + live (the full set streams, and the route echoes the instant on ``X-As-Of``); a far-past instant sees + no live mapping data (the annotatable set is empty). Either way the route returns 200 — ``as_of`` is a + filter, so an empty match is an empty collection, never a 404. This pins the wiring so the set + enumeration and the per-variant annotation cannot drift to different instants. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/{annotation_type}" + + # Far future: the seeded mapping records are live, so the whole set streams and X-As-Of echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + # Far past: nothing was live yet, so the annotatable set is empty — an empty 200 stream, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + + assert past_response.status_code == 200 + assert datetime.fromisoformat(past_response.headers["X-As-Of"]) == past + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] # Tests that annotated variants of the correct type are returned when appropriate. The contents of these @@ -4582,6 +5338,7 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) # The contents of the annotated variants objects should be tested in more detail elsewhere. response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") @@ -4613,6 +5370,10 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_thre data_files / "scores.csv", ) + # Variants are mapped on the allele substrate, so they are annotatable; the null annotation comes from + # the missing calibration/thresholds, not from missing mapping. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4636,6 +5397,9 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_cali data_files / "scores.csv", ) + # Variants are mapped on the allele substrate (so annotatable); the null comes from the absent calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4674,21 +5438,21 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set_when_some_vari client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") + assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") @pytest.mark.parametrize( @@ -4717,6 +5481,7 @@ def test_get_annotated_functional_impact_statement_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4750,6 +5515,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_calib }, ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent promoted calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4773,6 +5541,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_thres data_files / "scores.csv", ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent calibration/ranges. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4811,21 +5582,20 @@ def test_get_annotated_functional_impact_statement_for_score_set_when_some_varia client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "Statement" + assert annotated_variant.get("type") == "Statement" @pytest.mark.parametrize( @@ -4844,6 +5614,7 @@ def test_get_annotated_functional_study_result_for_score_set( experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4876,6 +5647,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4908,6 +5680,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_ranges_not_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4931,6 +5704,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4964,21 +5738,20 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian }, ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" + assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" def test_annotation_stream_reports_a_failing_variant_instead_of_truncating( @@ -4986,9 +5759,10 @@ def test_annotation_stream_reports_a_failing_variant_instead_of_truncating( ): """One variant that cannot be annotated must not cost the consumer the rest of the download.""" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_annotation_substrate(session, score_set) failing_annotation = create_failing_side_effect( # Representative of lib/annotation/util.py, which raises this on an unrecognized VRS Allele state. @@ -5020,11 +5794,10 @@ def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( ): """Every line is a variant record, so a body shorter than X-Total-Count is a truncated one.""" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - unmapped_variant = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) - assert unmapped_variant is not None + unmapped_variant_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn failing_annotation = create_failing_side_effect( ValueError("Unsupported VRS state type"), variant_study_result, fail_on_call=2 @@ -5033,19 +5806,19 @@ def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( with patch("mavedb.routers.score_sets.variant_study_result", failing_annotation): response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") + # X-Total-Count counts annotatable variants; the unmapped one is not among them. total_count = int(response.headers["X-Total-Count"]) - assert total_count == score_set["numVariants"] + assert total_count == score_set["numVariants"] - 1 response_data = parse_ndjson_response(response) + # The header and the body must agree — a shorter body is a truncated one. assert len(response_data) == total_count assert all("variant_urn" in record for record in response_data) + assert unmapped_variant_urn not in {record["variant_urn"] for record in response_data} - # A variant with no mapping data is an expected absence, distinguishable from the failure. + # The failure is reported in-band rather than by ending the stream early. errored = [record for record in response_data if "error" in record] - unannotated = [record for record in response_data if record["annotation"] is None and "error" not in record] assert len(errored) == 1 - assert len(unannotated) == 1 - assert unannotated[0]["variant_urn"] == unmapped_variant.urn ######################################################################################################################## @@ -5056,6 +5829,10 @@ def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( ######################################################################################################################## +#: Returned by the stubbed context factory; the stub annotation functions never read it. +_STUB_CONTEXT = object() + + class _StubAnnotation: def model_dump(self, **kwargs): return {"type": "Stub"} @@ -5069,37 +5846,72 @@ def model_dump(self, **kwargs): def _annotation_raising(exception): """An annotation function that fails.""" - def annotate(_mapped_variant): + def annotate(_context): raise exception return annotate @pytest.fixture -def mock_mapped_variant_for_stream(): - return create_mock_mapped_variant(clingen_allele_id="CA123456") +def stream_variant(): + """Stands in for the annotatable variant the stream iterates; only its URN is read here.""" + return SimpleNamespace(urn="tmp:test-urn#1") + + +def _record_for(variant, annotation_function, context=_STUB_CONTEXT): + """Drive one stream record with the chunk's loaded inputs stubbed out. + + ``context`` is what the inputs build for the variant; pass None for the variant whose mapping substrate + yields nothing to annotate. + """ + return _annotation_stream_record(SimpleNamespace(context_for=lambda _: context), variant, annotation_function) + + +def test_annotation_stream_record_reports_a_context_build_failure_as_an_error(stream_variant): + """Contexts are built per variant from a chunk's loaded inputs, so one variant failing to build stays + that variant's error record rather than ending the stream.""" + + def _broken(_): + raise ValueError("Unsupported VRS state type") + + record, outcome = _annotation_stream_record( + SimpleNamespace(context_for=_broken), stream_variant, lambda ctx: _StubAnnotation() + ) + + assert outcome == "errored" + assert record["error"] == {"type": "ValueError", "detail": "Unsupported VRS state type"} -def test_annotation_stream_record_serializes_a_successful_annotation(mock_mapped_variant_for_stream): - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _StubAnnotation()) +def test_annotation_stream_record_serializes_a_successful_annotation(stream_variant): + record, outcome = _record_for(stream_variant, lambda ctx: _StubAnnotation()) assert outcome == "annotated" - assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": {"type": "Stub"}} + assert record == {"variant_urn": stream_variant.urn, "annotation": {"type": "Stub"}} -def test_annotation_stream_record_treats_a_null_annotation_as_unannotated(mock_mapped_variant_for_stream): +def test_annotation_stream_record_treats_a_null_annotation_as_unannotated(stream_variant): """A variant the annotation layer declines to annotate is an expected outcome, not a failure.""" - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: None) + record, outcome = _record_for(stream_variant, lambda ctx: None) + + assert outcome == "unannotated" + assert record == {"variant_urn": stream_variant.urn, "annotation": None} + + +def test_annotation_stream_record_treats_an_absent_context_as_unannotated(stream_variant): + """No live mapping record at ``as_of`` means nothing to annotate, so the builder is never called.""" + record, outcome = _record_for( + stream_variant, _annotation_raising(AssertionError("must not be called")), context=None + ) assert outcome == "unannotated" - assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": None} + assert record == {"variant_urn": stream_variant.urn, "annotation": None} -def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(mock_mapped_variant_for_stream): +def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(stream_variant): # Preserved deliberately: a missing mapping is an expected absence, and reporting it as an error would # tell consumers a variant failed when nothing went wrong. - record, outcome = _annotation_stream_record( - mock_mapped_variant_for_stream, _annotation_raising(MappingDataDoesntExistException("no post-mapped allele")) + record, outcome = _record_for( + stream_variant, _annotation_raising(MappingDataDoesntExistException("no post-mapped allele")) ) assert outcome == "unannotated" @@ -5118,22 +5930,22 @@ def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(moc IndexError("list index out of range"), ], ) -def test_annotation_stream_record_reports_any_other_failure_as_an_error(mock_mapped_variant_for_stream, exception): - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, _annotation_raising(exception)) +def test_annotation_stream_record_reports_any_other_failure_as_an_error(stream_variant, exception): + record, outcome = _record_for(stream_variant, _annotation_raising(exception)) assert outcome == "errored" - assert record["variant_urn"] == mock_mapped_variant_for_stream.variant.urn + assert record["variant_urn"] == stream_variant.urn assert record["annotation"] is None assert record["error"] == {"type": type(exception).__name__, "detail": str(exception)} -def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(mock_mapped_variant_for_stream): +def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(stream_variant): """An emitted object that no longer dumps is the shape-dependent failure this stream must survive. Commit 5c155f4d fixed exactly this: a required field combined with `exclude_none` produced an object that built successfully and then failed on the way out. """ - record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _UndumpableAnnotation()) + record, outcome = _record_for(stream_variant, lambda ctx: _UndumpableAnnotation()) assert outcome == "errored" assert record["error"] == {"type": "ValueError", "detail": "Extension.value is required"} @@ -5146,19 +5958,32 @@ def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(mo def test_can_fetch_current_gnomad_variants_for_score_set(client, setup_router_db, session, data_provider, data_files): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + # gnomAD variant id=1 (seeded by setup_router_db) links to the first variant's authoritative allele. + variants = seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) + expected_urn = variants[0].urn + expected_digest = f"csv-auth-{variants[0].id}" response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants") assert response.status_code == 200 + assert response.headers["x-as-of"] == "current" response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 + assert variant_links[0]["variantUrn"] == expected_urn + assert variant_links[0]["alleleDigest"] == expected_digest gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -5167,10 +5992,17 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version={TEST_GNOMAD_DATA_VERSION}") assert response.status_code == 200 @@ -5178,8 +6010,8 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -5188,10 +6020,17 @@ def test_cannot_fetch_current_gnomad_variants_for_score_set_with_nonexistent_ver client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version=nonexistent_version") assert response.status_code == 404 @@ -5208,10 +6047,17 @@ def test_cannot_fetch_gnomad_variants_for_nonexistent_score_set( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/gnomad-variants") @@ -5224,7 +6070,7 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) @@ -5238,6 +6084,174 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( ) +def _seed_lean_mapping(session, variant_urn): + """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose + authoritative allele carries a digest + ClinGen id + a live VEP consequence, plus its canonical + genomic projection sibling (shared ``projection_group``) and the protein apex — the full triple.""" + measured = GroupSpec( + hgvs_c="NM_000546.6:c.1216G>A", hgvs_g="NC_000017.11:g.7676154C>T", hgvs_p="NP_000537.3:p.Ala406Thr" + ) + seed_mapping_record( + session, + variant_urn, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + alleles=[ + AlleleSpec( + digest="cdna-digest", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + vep_consequence="missense_variant", + group=measured, + ), + AlleleSpec( + digest="gen-digest", + level="genomic", + hgvs_g="NC_000017.11:g.7676154C>T", + group=measured, + ), + AlleleSpec(digest="prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr"), + ], + ) + + +def test_get_lean_variants(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_lean_mapping(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + records = response.json() + # All three variants are returned, ordered by variant number. + assert [r["variantUrn"] for r in records] == [f"{score_set['urn']}#{n}" for n in (1, 2, 3)] + for record in records: + LeanVariant.model_validate_json(json.dumps(record)) + + mapped = records[0] + assert mapped["score"] == 0.3 + assert mapped["consequence"] == "missense_variant" + assert mapped["clingenAlleleId"] == "CA123" + assert mapped["assayLevelDigest"] == "cdna-digest" + # Submitted HGVS (from the uploaded CSV), each with its parsed block riding alongside. + assert mapped["hgvsNt"] == {"hgvs": "c.1A>T", "position": 1, "ref": "A", "alt": "T"} + assert mapped["hgvsPro"] == {"hgvs": "p.Thr1Ser", "position": 1, "ref": "Thr", "alt": "Ser"} + # The mapped triple (reference frame): the measured slot named by assayLevel, its projection_group + # genomic sibling, and the protein apex. mapped.cdna is the search key even here (measured at cdna). + assert mapped["assayLevel"] == "cdna" + assert mapped["mapped"]["cdna"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} + assert mapped["mapped"]["genomic"] == { + "hgvs": "NC_000017.11:g.7676154C>T", + "position": 7676154, + "ref": "C", + "alt": "T", + } + assert mapped["mapped"]["protein"] == { + "hgvs": "NP_000537.3:p.Ala406Thr", + "position": 406, + "ref": "Ala", + "alt": "Thr", + } + + # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none), + # and the mapped triple serializes empty (no slots) since none is populated. + unmapped = records[1] + assert unmapped["score"] == 1.0 + assert unmapped["hgvsNt"]["hgvs"] == "c.2C>T" + for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevel"): + assert omitted not in unmapped + assert unmapped["mapped"] == {} + + +def test_get_lean_variants_unknown_score_set_is_404(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variants") + assert response.status_code == 404 + + +def test_get_lean_variants_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Default is current — the resolved content-time is echoed for the client. + current = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + + # An explicit as_of is accepted and echoed back; before mapping exists everything is unmapped. + historical = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants", params={"as_of": "2020-01-01T00:00:00Z"}) + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert all("assayLevelDigest" not in record for record in historical.json()) + + +def test_get_lean_variants_other_user_cannot_read_private(client, session, data_provider, data_files, setup_router_db): + """The endpoint gates on READ: a non-owner cannot read another user's private (unpublished) set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + assert f"score set with URN '{score_set['urn']}' not found" in response.json()["detail"] + + +def test_get_lean_variants_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + + +def test_get_lean_variants_anonymous_can_read_published( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + """A published score set is world-readable, so an anonymous caller gets the full lean set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + published = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 + + +def test_get_lean_variants_admin_can_read_private( + client, session, data_provider, data_files, setup_router_db, admin_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(admin_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 + + @pytest.mark.parametrize( "mock_publication_fetch", [ @@ -5300,3 +6314,98 @@ def test_publish_returns_the_owners_own_private_calibration( published = publish_score_set(client, score_set["urn"]) assert [c["urn"] for c in (published.get("scoreCalibrations") or [])] == [calibration["urn"]] + + +def test_patch_with_invalid_data_on_unreadable_score_set_is_404(session, client, setup_router_db): + """Validation errors echo the stored record, so a caller who may not update the score set must be + refused before validation: a 422 here would confirm that a private score set exists.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.patch(f"/api/v1/score-sets-with-variants/{score_set['urn']}", data={"title": json.dumps(None)}) + + assert response.status_code == 404 + + +def test_meta_analyses_are_listed_only_to_callers_who_may_read_them( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, extra_user_app_overrides +): + """Anyone may start a meta-analysis of a published score set, so the score set must not name a private + one to callers who can't read it.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + meta_analysis = create_seq_score_set( + client, None, update={"title": "Private meta-analysis", "metaAnalyzesScoreSetUrns": [published["urn"]]} + ) + assert client.get(f"/api/v1/score-sets/{published['urn']}").json()["metaAnalyzedByScoreSetUrns"] == [ + meta_analysis["urn"] + ] + + assert client.get(f"/api/v1/score-sets/{published['urn']}").json()["metaAnalyzedByScoreSetUrns"] == [] + with DependencyOverrider(anonymous_app_overrides): + assert client.get(f"/api/v1/score-sets/{published['urn']}").json()["metaAnalyzedByScoreSetUrns"] == [] + experiment_score_sets = client.get(f"/api/v1/experiments/{published['experiment']['urn']}/score-sets").json() + assert [ss["metaAnalyzedByScoreSetUrns"] for ss in experiment_score_sets] == [[]] + + +def test_official_collections_list_only_readable_collections_and_members( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides +): + """An official collection badges every member, so a public record's response must not reveal the private + members it shares a badge with, or a private badge collection at all.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + published_experiment_urn = published["experiment"]["urn"] + private_score_set = create_seq_score_set(client, published_experiment_urn) + private_experiment = create_experiment(client) + + public_badge = create_collection(client) + private_badge = create_collection(client) + for collection, private in ((public_badge, False), (private_badge, True)): + for member in (published, private_score_set): + response = client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", json={"score_set_urn": member["urn"]} + ) + assert response.status_code == 200, response.text + for experiment_urn in (published_experiment_urn, private_experiment["urn"]): + response = client.post( + f"/api/v1/collections/{collection['urn']}/experiments", json={"experiment_urn": experiment_urn} + ) + assert response.status_code == 200, response.text + session.query(CollectionDbModel).filter(CollectionDbModel.urn == collection["urn"]).update( + {"badge_name": "Badge", "private": private} + ) + session.commit() + + def members(collections): + return {c["urn"]: (sorted(c["scoreSetUrns"]), sorted(c["experimentUrns"])) for c in collections} + + everything = ( + sorted([published["urn"], private_score_set["urn"]]), + sorted([published_experiment_urn, private_experiment["urn"]]), + ) + owner_view = client.get(f"/api/v1/score-sets/{published['urn']}").json() + assert members(owner_view["officialCollections"]) == { + public_badge["urn"]: everything, + private_badge["urn"]: everything, + } + + with DependencyOverrider(anonymous_app_overrides): + anonymous_view = client.get(f"/api/v1/score-sets/{published['urn']}").json() + experiment_view = client.get(f"/api/v1/experiments/{published_experiment_urn}").json() + experiment_list = client.get("/api/v1/experiments/").json() + + expected = {public_badge["urn"]: ([published["urn"]], [published_experiment_urn])} + assert members(anonymous_view["officialCollections"]) == expected + assert members(anonymous_view["experiment"]["officialCollections"]) == expected + assert members(experiment_view["officialCollections"]) == expected + assert [members(e["officialCollections"]) for e in experiment_list] == [expected] diff --git a/tests/routers/test_seqrepo.py b/tests/routers/test_seqrepo.py index 231f06a5e..c54bd1040 100644 --- a/tests/routers/test_seqrepo.py +++ b/tests/routers/test_seqrepo.py @@ -3,6 +3,8 @@ import pytest +pytestmark = pytest.mark.unit + arq = pytest.importorskip("arq") cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") @@ -46,12 +48,55 @@ def test_get_sequence_multiple_ids(client): assert "Multiple sequences exist" in resp.text +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_start(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + start = 1 + resp = client.get(f"/api/v1/seqrepo/sequence/{alias}?start={start}") + assert resp.status_code == 200 + assert resp.text == metadata["seq"][start:] + + +@pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) +def test_get_sequence_only_end(client, entry): + alias = list(entry.keys())[0] + metadata = list(entry.values())[0] + end = 3 + resp = client.get(f"/api/v1/seqrepo/sequence/{alias}?end={end}") + assert resp.status_code == 200 + assert resp.text == metadata["seq"][:end] + + def test_get_sequence_invalid_coords(client): resp = client.get(f"/api/v1/seqrepo/sequence/{VALID_ENSEMBL_IDENTIFIER}?start=10&end=5") assert resp.status_code == 422 assert "Invalid coordinates" in resp.text +# Coordinates outside the sequence used to stream a truncated body under a 200 instead of being rejected. +@pytest.mark.parametrize( + "query", + [ + "start=10&end=12", + "start=1&end=12", + "start=10", + "start=-1&end=2", + ], +) +def test_get_sequence_coords_outside_sequence(client, query): + resp = client.get(f"/api/v1/seqrepo/sequence/{VALID_ENSEMBL_IDENTIFIER}?{query}") + assert resp.status_code == 422 + assert "Invalid coordinates" in resp.text + + +def test_get_sequence_only_end_too_large_clamps(client): + # Mirrors refget.py: an over-long lone end is clamped rather than rejected. + resp = client.get(f"/api/v1/seqrepo/sequence/{VALID_ENSEMBL_IDENTIFIER}?end=12") + assert resp.status_code == 200 + assert resp.text == "GGGG" + + @pytest.mark.parametrize("entry", TEST_SEQREPO_INITIAL_STATE) def test_get_metadata_success(client, entry): alias = list(entry.keys())[0] @@ -85,3 +130,12 @@ def test_get_versions(client): resp = client.get("/api/v1/seqrepo/version") assert resp.status_code == 200 assert isinstance(resp.json(), dict) + + +# The shared SeqRepo holds the target sequences of unpublished score sets, so lookups must not match by +# wildcard or digest prefix. +@pytest.mark.parametrize("path", ["sequence/{alias}", "metadata/{alias}"]) +@pytest.mark.parametrize("alias", ["ga4gh:SQ.%25", "MD5:%25", "%25", "0123456789"]) +def test_lookup_does_not_match_wildcards_or_digest_prefixes(client, path, alias): + resp = client.get(f"/api/v1/seqrepo/{path.format(alias=alias)}") + assert resp.status_code == 404 diff --git a/tests/routers/test_statistics.py b/tests/routers/test_statistics.py index 69be6ffba..21ae62676 100644 --- a/tests/routers/test_statistics.py +++ b/tests/routers/test_statistics.py @@ -21,7 +21,12 @@ TEST_PUBMED_IDENTIFIER, VALID_GENE, ) -from tests.helpers.util.score_set import publish_score_set, create_acc_score_set, create_seq_score_set +from tests.helpers.util.score_set import ( + publish_score_set, + create_acc_score_set, + create_seq_score_set, + seed_annotation_substrate, +) from tests.helpers.util.experiment import create_experiment from tests.helpers.util.variant import mock_worker_variant_insertion, create_mapped_variants_for_score_set @@ -65,7 +70,12 @@ def setup_seq_scoreset(setup_router_db, session, data_provider, client, data_fil unpublished_score_set = mock_worker_variant_insertion( client, session, data_provider, unpublished_score_set, data_files / "scores.csv" ) + # Fully-mapped state: target post_mapped_metadata + mapping_state (read by the target-gene + # statistics). MappedVariant rows are seeded here too but are no longer read by the MV. create_mapped_variants_for_score_set(session, unpublished_score_set["urn"], TEST_MINIMAL_MAPPED_VARIANT) + # The published_variants MV reads the MappingRecord/Allele substrate now (not MappedVariant), so + # give each variant a live mapping record for the mapped-variant statistics to count. + seed_annotation_substrate(session, unpublished_score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: publish_score_set(client, unpublished_score_set["urn"]) diff --git a/tests/routers/test_urn_redirects.py b/tests/routers/test_urn_redirects.py new file mode 100644 index 000000000..cdeeb095a --- /dev/null +++ b/tests/routers/test_urn_redirects.py @@ -0,0 +1,154 @@ +# ruff: noqa: E402 + +from urllib.parse import quote, unquote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.experiment import Experiment as ExperimentDbModel +from mavedb.models.experiment_set import ExperimentSet as ExperimentSetDbModel +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel + +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import ( + create_seq_score_set, + create_seq_score_set_with_variants, + publish_score_set, +) + +UNKNOWN_TMP_URN = "tmp:00000000-0000-4000-8000-00000000ffff" + +COLLECTIONS = {"experiment_set": "experiment-sets", "experiment": "experiments", "score_set": "score-sets"} + + +@pytest.fixture +def published(session, data_provider, client, setup_router_db, data_files): + """Publish a score set, and report what each of its records was called before and after.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + published_score_set = publish_score_set(client, score_set["urn"]) + + return { + "experiment_set": (experiment["experimentSetUrn"], published_score_set["experiment"]["experimentSetUrn"]), + "experiment": (experiment["urn"], published_score_set["experiment"]["urn"]), + "score_set": (score_set["urn"], published_score_set["urn"]), + } + + +@pytest.mark.integration +class TestForwardingRetiredUrns: + @pytest.mark.parametrize("record", ["experiment_set", "experiment", "score_set"]) + def test_retired_urn_is_forwarded_to_the_published_record(self, client, published, record): + """Publication renames all three records, so a stale link to any of them has to resolve.""" + retired_urn, published_urn = published[record] + collection = COLLECTIONS[record] + + response = client.get(f"/api/v1/{collection}/{retired_urn}", follow_redirects=False) + + assert response.status_code == 308 + assert response.headers["location"] == f"/api/v1/{collection}/{published_urn}" + + @pytest.mark.parametrize("record", ["experiment_set", "experiment", "score_set"]) + def test_a_client_following_the_forward_reaches_the_record(self, client, published, record): + retired_urn, published_urn = published[record] + collection = COLLECTIONS[record] + + response = client.get(f"/api/v1/{collection}/{retired_urn}") + + assert response.status_code == 200 + assert response.json()["urn"] == published_urn + + def test_a_sub_resource_of_a_retired_urn_is_forwarded(self, client, published): + """The complaint in mavedb-ui#617 is about a page, which loads more than the record itself.""" + retired_urn, published_urn = published["score_set"] + + response = client.get(f"/api/v1/score-sets/{retired_urn}/scores", follow_redirects=False) + + assert response.status_code == 308 + assert response.headers["location"] == f"/api/v1/score-sets/{published_urn}/scores" + + def test_a_variant_of_a_retired_score_set_is_forwarded(self, client, published): + """A variant URN is derived from its score set's, and needs no redirect of its own.""" + retired_urn, published_urn = published["score_set"] + + variant_path = f"/api/v1/variants/{quote(f'{retired_urn}#1', safe='')}/csv-namespaces" + + response = client.get(variant_path, follow_redirects=False) + + assert response.status_code == 308 + location = response.headers["location"] + assert unquote(location) == f"/api/v1/variants/{published_urn}#1/csv-namespaces" + # The '#' has to come back encoded, or a client reads the rest of the location as a fragment. + assert "%23" in location + assert client.get(variant_path).status_code == 200 + + def test_a_query_string_survives_forwarding(self, client, published): + retired_urn, published_urn = published["score_set"] + + response = client.get(f"/api/v1/score-sets/{retired_urn}/scores?start=0&limit=1", follow_redirects=False) + + assert response.status_code == 308 + assert response.headers["location"] == f"/api/v1/score-sets/{published_urn}/scores?start=0&limit=1" + + def test_a_live_temporary_urn_is_served_not_forwarded(self, client, setup_router_db): + """An unpublished record still answers to the temporary URN it was created with.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}", follow_redirects=False) + + assert response.status_code == 200 + assert response.json()["urn"] == score_set["urn"] + + def test_an_unknown_temporary_urn_is_still_a_404(self, client, setup_router_db): + response = client.get(f"/api/v1/score-sets/{UNKNOWN_TMP_URN}", follow_redirects=False) + + assert response.status_code == 404 + + @pytest.mark.parametrize( + "record,model", + [ + ("score_set", ScoreSetDbModel), + ("experiment", ExperimentDbModel), + ("experiment_set", ExperimentSetDbModel), + ], + ) + def test_a_retired_urn_is_not_forwarded_to_a_private_record(self, session, client, published, record, model): + """A Location header names its target, to an anonymous caller, before any route checks anything. + + Publication only records a redirect onto a record it is making public, and nothing in the + application returns a published record to private, so this state is reached here the only way + it could be reached in production: out of band. + """ + retired_urn, published_urn = published[record] + session.query(model).filter(model.urn == published_urn).one().private = True + session.commit() + + response = client.get(f"/api/v1/{COLLECTIONS[record]}/{retired_urn}", follow_redirects=False) + + assert response.status_code == 404 + assert published_urn not in response.text + + def test_a_retired_urn_is_not_forwarded_to_a_deleted_record(self, session, client, published): + """A deleted record leaves its redirect row behind, pointing at a URN that resolves to nothing.""" + retired_urn, published_urn = published["score_set"] + session.delete(session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == published_urn).one()) + session.commit() + + response = client.get(f"/api/v1/score-sets/{retired_urn}", follow_redirects=False) + + assert response.status_code == 404 + assert published_urn not in response.text + + def test_a_write_to_a_retired_urn_is_not_forwarded(self, client, published): + """Forwarding a stale publish request would rename the live public record it reached.""" + retired_urn, _ = published["score_set"] + + response = client.post(f"/api/v1/score-sets/{retired_urn}/publish", follow_redirects=False) + + assert response.status_code == 404 diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 48f3fbd99..944c4b1a1 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -1,6 +1,15 @@ # ruff: noqa: E402 +"""Router tests for the assayed variant-detail endpoint (``GET /variants/{urn}``) and the +single-variant CSV export (``GET /variants/{urn}/csv``). + +Exercise the HTTP surface end to end: the two-tier envelope serializes and validates, the ``as_of`` +content-time is echoed in the ``X-As-Of`` header, a superseded variant self-describes rather than +reading as current, the READ gate holds for private score sets, and a private calibration is never +served over either surface. +""" import csv +from datetime import datetime, timezone from io import StringIO from unittest.mock import patch from urllib.parse import quote @@ -11,16 +20,257 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") -from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from sqlalchemy import select +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.view_models.variant_detail import VariantDetail from tests.helpers.dependency_overrider import DependencyOverrider from tests.helpers.util.experiment import create_experiment from tests.helpers.util.score_set import ( - create_seq_score_set_with_mapped_variants, - link_clinvar_control_to_mapped_variant, + create_seq_score_set_with_variants, + link_clinical_controls_to_alleles, publish_score_set, + seed_csv_substrate, ) +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this on the detail path.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed_mapping(session, variant_urn): + """Give a variant a live coding-measured mapping record whose authoritative allele carries a + digest + ClinGen id + a live VEP consequence, its genomic projection sibling (shared + projection_group), a protein apex, and a synonymous cousin (a nt encoder of the same consequence in + a different projection group) — enough to build Cat-VRS and exercise the projection + convergent axes.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele( + vrs_digest="cdna-digest", + level="cdna", + post_mapped=_post_mapped(), + clingen_allele_id="CA123", + hgvs_c="NM_000546.6:c.1216G>A", + ) + genomic = Allele( + vrs_digest="gen-digest", level="genomic", post_mapped=_post_mapped(), hgvs_g="NC_000017.11:g.7676154C>T" + ) + protein = Allele( + vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" + ) + cousin = Allele( + vrs_digest="cousin-digest", level="cdna", post_mapped=_post_mapped(), hgvs_c="NM_000546.6:c.1218C>T" + ) + measured_group = ProjectionGroup( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + hgvs_c=measured.hgvs_c, + hgvs_g=genomic.hgvs_g, + hgvs_p=protein.hgvs_p, + ) + cousin_group = ProjectionGroup(mapping_record_id=record.id, score_set_id=record.score_set_id, hgvs_c=cousin.hgvs_c) + session.add_all([measured, genomic, protein, cousin, measured_group, cousin_group]) + session.commit() + + session.add_all( + [ + # The measured group holds the measured cdna, its genomic projection, and the protein apex; the + # cousin sits in its own projection group. + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=measured.id, + is_authoritative=True, + projection_group=measured_group.id, + ), + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=genomic.id, + is_authoritative=False, + projection_group=measured_group.id, + ), + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=protein.id, + is_authoritative=False, + projection_group=measured_group.id, + ), + MappingRecordAllele( + mapping_record_id=record.id, + score_set_id=record.score_set_id, + allele_id=cousin.id, + is_authoritative=False, + projection_group=cousin_group.id, + ), + VepAlleleConsequence( + allele_id=measured.id, + functional_consequence="missense_variant", + source_version="116", + access_date="2026-01-01", + ), + ] + ) + session.commit() + + +def test_get_variant_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_mapping(session, urn) + + response = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}") + + assert response.status_code == 200 + body = response.json() + VariantDetail.model_validate(body) + + assert body["urn"] == urn + assert body["assayLevel"] == "cdna" + assert body["referenceHgvs"] == "NM_000546.6:c.1216G>A" + assert body["assayLevelDigest"] == "cdna-digest" + assert body["clingenAlleleId"] == "CA123" + assert body["mode"] == "projection" + assert {level: slot["hgvs"] for level, slot in body["mapped"].items() if slot is not None} == { + "cdna": "NM_000546.6:c.1216G>A", + "genomic": "NC_000017.11:g.7676154C>T", + "protein": "NP_000537.3:p.Ala406Thr", + } + # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). + assert body["molecularRepresentation"]["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar (every linked allele is a + # member, the cousin included), so the envelope is internally consistent. + assert len(body["molecularRepresentation"]["members"]) == len(body["alleles"]) + # The alleles identity sidecar serializes camelCase, keyed by digest; the protein member is a + # translation of the defining coding allele (which itself has no relation to itself). + assert body["alleles"]["cdna-digest"]["level"] == "cdna" + assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" + assert ( + body["alleles"]["cdna-digest"]["relation"] is None + ) # measured allele: no relation to itself (null, not dropped) + # The measured allele is the focus (isFocus); it carries no derivation (that axis describes the + # *other* members relative to it), and pairs with its genomic projection sibling (projectionOf). + assert body["alleles"]["cdna-digest"]["isFocus"] is True + assert body["alleles"]["cdna-digest"]["derivation"] is None # null, not dropped (exclude_none=False) + assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" + assert body["alleles"]["gen-digest"]["isFocus"] is False + assert body["alleles"]["gen-digest"]["relation"] == "coordinate_representation_of" + assert body["alleles"]["gen-digest"]["derivation"] == "projection" + assert body["alleles"]["gen-digest"]["projectionOf"] == "cdna-digest" + assert body["alleles"]["prot-digest"]["level"] == "protein" + assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" + assert body["alleles"]["prot-digest"]["relation"] == "translation_of" + # The apex is a deterministic projection, but not a c<->g counterpart, so it pairs with nothing. + assert body["alleles"]["prot-digest"]["derivation"] == "projection" + assert body["alleles"]["prot-digest"]["projectionOf"] is None + # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is + # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). + assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes" + assert body["alleles"]["cousin-digest"]["derivation"] == "convergent" + assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" + assert body["isCurrent"] is True + assert body["supersededByScoreSet"] is None # null when current (stable envelope, not dropped) + assert response.headers["X-As-Of"] == "current" + + +def test_get_variant_detail_unknown_urn_is_404(client, setup_router_db): + response = client.get("/api/v1/variants/urn:mavedb:00000000-a-1#1") + assert response.status_code == 404 + + +def test_get_variant_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + + historical = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + # Before any mapping existed, the molecular layer is empty — the stable envelope carries it as null. + assert historical.json()["molecularRepresentation"] is None + + +def test_superseded_variant_self_describes(client, session, data_provider, data_files, setup_router_db): + """A variant on a superseded score set is still served, but flags its version standing.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Link newer as the version that replaces older (replaces_id / superseding relationship). + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + + response = client.get(f"/api/v1/variants/{older['urn']}%231") + + assert response.status_code == 200 + body = response.json() + assert body["isCurrent"] is False + assert body["supersededByScoreSet"] == newer["urn"] + + +def test_get_variant_detail_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/variants/{score_set['urn']}%231") + + assert response.status_code == 404 def _first_variant_urn(session, score_set_urn): @@ -33,11 +283,12 @@ def _csv_path(variant_urn): return f"/api/v1/variants/{quote(variant_urn, safe='')}/csv" -def _published_score_set_with_mapped_variants(client, session, data_provider, data_files): +def _published_score_set_with_mappings(client, session, data_provider, data_files): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_csv_substrate(session, score_set, assay_level="genomic", hgvs_g="NC_000010.11:g.1A>G") with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published = publish_score_set(client, score_set["urn"]) return published @@ -45,7 +296,7 @@ def _published_score_set_with_mapped_variants(client, session, data_provider, da class TestGetVariantCsv: def test_returns_csv_attachment(self, session, data_provider, client, setup_router_db, data_files): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(_csv_path(variant_urn)) @@ -55,7 +306,7 @@ def test_returns_csv_attachment(self, session, data_provider, client, setup_rout assert response.headers["content-disposition"] == f'attachment; filename="{variant_urn}.csv"' def test_reports_the_requested_variant(self, session, data_provider, client, setup_router_db, data_files): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(_csv_path(variant_urn)) @@ -67,7 +318,7 @@ def test_reports_the_requested_variant(self, session, data_provider, client, set assert rows[0]["relationship.match_type"] == "exact" def test_namespaces_restrict_the_columns(self, session, data_provider, client, setup_router_db, data_files): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"{_csv_path(variant_urn)}?namespaces=scores") @@ -82,10 +333,10 @@ def test_clinvar_namespace_is_labeled_with_its_release( self, session, data_provider, client, setup_router_db, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + link_clinical_controls_to_alleles(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published = publish_score_set(client, score_set["urn"]) variant_urn = _first_variant_urn(session, published["urn"]) @@ -104,7 +355,7 @@ def test_invalid_namespace_is_rejected( self, session, data_provider, client, setup_router_db, data_files, namespace ): """FastAPI validates the namespace vocabulary from the parameter type, before the handler runs.""" - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") @@ -115,7 +366,7 @@ def test_invalid_namespace_is_rejected( @pytest.mark.parametrize("namespace", ["scores", "clinvar.2024_01"]) def test_valid_namespace_is_accepted(self, session, data_provider, client, setup_router_db, data_files, namespace): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") @@ -157,7 +408,7 @@ def test_private_score_set_is_not_readable_by_other_users( self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) variant_urn = _first_variant_urn(session, score_set["urn"]) @@ -176,6 +427,7 @@ class TestPrivateCalibrationsAreNotServedOverHttp: """ def _private_calibration(self, session, score_set_urn): + from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() @@ -188,6 +440,7 @@ def _private_calibration(self, session, score_set_urn): primary=False, private=True, calibration_metadata={}, + disease_term=get_generic_disease_term(session), created_by_id=score_set.created_by_id, modified_by_id=score_set.modified_by_id, ) @@ -198,7 +451,7 @@ def _private_calibration(self, session, score_set_urn): def test_another_user_naming_the_urn_gets_no_interpretation( self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) calibration_urn = self._private_calibration(session, published["urn"]) namespace = f"calibration.{calibration_urn}" @@ -215,7 +468,7 @@ def test_another_user_naming_the_urn_gets_no_interpretation( def test_another_user_is_not_offered_it_by_discovery( self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) calibration_urn = self._private_calibration(session, published["urn"]) with DependencyOverrider(extra_user_app_overrides): @@ -225,13 +478,77 @@ def test_another_user_is_not_offered_it_by_discovery( assert f"calibration.{calibration_urn}" not in [entry["namespace"] for entry in response.json()] +class TestVariantCsvAsOf: + """`as_of` time-travels the single-variant CSV, as it does the score-set CSV. + + The library call has always supported it; the route did not expose it, so the whole reconstruction + path was unreachable over HTTP and the response did not even self-describe its content-time. + """ + + def _seeded(self, client, session, data_provider, data_files, seeded_at): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000010.11:g.1A>G", + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + return publish_score_set(client, score_set["urn"]) + + def test_csv_echoes_and_reconstructs_at_the_requested_instant( + self, session, data_provider, client, setup_router_db, data_files + ): + published = self._seeded(client, session, data_provider, data_files, datetime(2020, 1, 1, tzinfo=timezone.utc)) + variant_urn = _first_variant_urn(session, published["urn"]) + + current = client.get(_csv_path(variant_urn), params={"namespaces": "vep"}) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + assert any( + row["vep.vep_functional_consequence"] == "missense_variant" + for row in csv.DictReader(StringIO(current.text)) + ) + + # Before the mapping was live, the same columns reconstruct as NA. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(_csv_path(variant_urn), params={"namespaces": "vep", "as_of": past.isoformat()}) + assert before.status_code == 200 + assert before.headers["X-As-Of"] == past.isoformat() + assert all(row["vep.vep_functional_consequence"] == "NA" for row in csv.DictReader(StringIO(before.text))) + + def test_namespace_discovery_answers_for_the_requested_instant( + self, session, data_provider, client, setup_router_db, data_files + ): + """Discovery and download must agree, or the picker offers columns the download returns NA.""" + published = self._seeded(client, session, data_provider, data_files, datetime(2020, 1, 1, tzinfo=timezone.utc)) + variant_urn = _first_variant_urn(session, published["urn"]) + url = f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces" + + current = client.get(url) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + assert "vep" in {entry["namespace"] for entry in current.json()} + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(url, params={"as_of": past.isoformat()}) + assert before.status_code == 200 + assert before.headers["X-As-Of"] == past.isoformat() + assert "vep" not in {entry["namespace"] for entry in before.json()} + + class TestCsvNamespaceDiscovery: """The discovery endpoints advertise what a namespace picker should offer.""" def test_score_set_namespaces_are_labeled_and_grouped( self, session, data_provider, client, setup_router_db, data_files ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") @@ -247,7 +564,7 @@ def test_score_set_namespaces_are_labeled_and_grouped( def test_score_set_detail_response_is_unchanged(self, session, data_provider, client, setup_router_db, data_files): """Discovery is its own request, so it must not appear on the score-set page's critical path.""" - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) response = client.get(f"/api/v1/score-sets/{published['urn']}") @@ -257,7 +574,7 @@ def test_score_set_detail_response_is_unchanged(self, session, data_provider, cl def test_variant_namespaces_are_labeled_and_grouped( self, session, data_provider, client, setup_router_db, data_files ): - published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + published = _published_score_set_with_mappings(client, session, data_provider, data_files) variant_urn = _first_variant_urn(session, published["urn"]) response = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces") @@ -273,10 +590,10 @@ def test_advertised_namespaces_are_accepted_by_the_csv_endpoints( ): """Discovery and validation must agree, or the picker offers options that 422.""" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + link_clinical_controls_to_alleles(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): published = publish_score_set(client, score_set["urn"]) diff --git a/tests/routers/test_variant_annotation.py b/tests/routers/test_variant_annotation.py new file mode 100644 index 000000000..60c9bebad --- /dev/null +++ b/tests/routers/test_variant_annotation.py @@ -0,0 +1,477 @@ +# ruff: noqa: E402 +"""Router tests for the variant annotation surface relocated onto the Allele substrate. + +The VA-Spec statement endpoints (``/variants/{urn}/va/*``) and the VRS-identifier lookup +(``/variants/vrs/{identifier}``) moved off the legacy ``/mapped-variants`` router in #743; they build +from the ``MappingRecord`` / ``Allele`` substrate (seeded by ``seed_annotation_substrate``), never the +legacy ``MappedVariant``. +""" + +import json +from datetime import datetime, timezone +from unittest.mock import patch + +import pytest + +from tests.helpers.util.user import change_ownership + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote_plus + +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement + +from sqlalchemy import select + +from mavedb.models.mapping_record import MappingRecord as MappingRecordDbModel +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_GA4GH_IDENTIFIER, + TEST_PUBMED_IDENTIFIER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + create_seq_score_set_with_variants, + publish_score_set, + seed_annotation_substrate, +) + +_PUBLICATION_FETCH = [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +# --- StudyResult ----------------------------------------------------------------------------------- + + +def test_variant_study_result(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." + ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) + + +def test_variant_study_result_404_when_variant_missing(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + urn = f"{score_set['urn']}#404" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert response.json()["detail"] == f"variant with URN '{urn}' not found" + + +def test_variant_study_result_404_when_no_mapping_data(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # The variant exists but has no mapping record seeded on the new substrate. + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert f"No study result exists for variant {urn}: no mapping data exists." in response.json()["detail"] + + +# --- Functional-impact Statement ------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_functional_impact_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # A calibration can be published only on a published score set. + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." + # The nested study result keeps its subclass fields, which serializing as the declared base type drops. + study_result = response_data["hasEvidenceLines"][0]["hasEvidenceItems"][0] + assert study_result["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + assert "focus" in study_result + Statement.model_validate_json(json.dumps(response_data)) + + +def test_variant_functional_impact_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # Mapped on the new substrate, but no (primary) calibration => insufficient evidence. + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/functional-statement") + + assert response.status_code == 404 + assert ( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- Pathogenicity Statement ----------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_pathogenicity_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # A calibration can be published only on a published score set. + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + score_set = publish_score_set(client, score_set["urn"]) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") + response_data = response.json() + + assert response.status_code == 200 + assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] + VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) + + +def test_variant_pathogenicity_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/pathogenicity-statement") + + assert response.status_code == 404 + assert ( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- VRS-identifier lookup ------------------------------------------------------------------------- + + +def test_lookup_variants_by_vrs_identifier(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # vrs_digest stores the full GA4GH VRS id in production (the mapper writes post_mapped["id"] into it), + # so the lookup matches the identifier against vrs_digest. The allele's ClinGen id rides alongside the URN. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + response_data = response.json() + + assert response.status_code == 200 + assert len(response_data) == 1 + assert response_data[0]["variantUrn"] == f"{score_set['urn']}#1" + assert response_data[0]["clingenAlleleId"] == "CA1" + assert response_data[0]["vrsId"] == TEST_GA4GH_IDENTIFIER + + +PRE_MAPPED_IDENTIFIER = "ga4gh:VA.PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP" + + +def _set_pre_mapped_digest(session, variant, digest): + record = session.scalar(select(MappingRecordDbModel).where(MappingRecordDbModel.variant_id == variant.id)) + record.vrs_digest = digest + session.commit() + + +def test_lookup_variants_by_pre_mapped_vrs_identifier(client, session, data_provider, data_files, setup_router_db): + """A pre-mapped identifier resolves to its variant, reported with the variant's measured allele.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + _set_pre_mapped_digest(session, variant, PRE_MAPPED_IDENTIFIER) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(PRE_MAPPED_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [ + { + "variantUrn": f"{score_set['urn']}#1", + "clingenAlleleId": "CA1", + "vrsId": TEST_GA4GH_IDENTIFIER, + "level": "cdna", + } + ] + + +def test_lookup_variants_by_pre_mapped_vrs_identifier_of_an_unmapped_variant( + client, session, data_provider, data_files, setup_router_db +): + """A variant that did not map still resolves by its pre-mapped identifier, with no allele fields.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record(session, variant, assay_level="cdna", alleles=[]) + _set_pre_mapped_digest(session, variant, PRE_MAPPED_IDENTIFIER) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(PRE_MAPPED_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [ + {"variantUrn": f"{score_set['urn']}#1", "clingenAlleleId": None, "vrsId": None, "level": None} + ] + + +def test_lookup_variants_by_vrs_identifier_reports_a_variant_once_when_pre_and_post_mapped_agree( + client, session, data_provider, data_files, setup_router_db +): + """An accession-based target's pre- and post-mapped variations can share an identifier.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True, clingen_allele_id="CA1") + ], + ) + _set_pre_mapped_digest(session, variant, TEST_GA4GH_IDENTIFIER) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert response.status_code == 200 + assert [match["variantUrn"] for match in response.json()] == [f"{score_set['urn']}#1"] + + +def test_lookup_variants_by_vrs_identifier_orders_matches_by_variant_number( + client, session, data_provider, data_files, setup_router_db +): + """A shared allele resolves to every variant linking it, in variant-number order whatever the seeding order.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + for number in (3, 1, 2): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{number}")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True)], + ) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert [match["variantUrn"] for match in response.json()] == [f"{score_set['urn']}#{n}" for n in (1, 2, 3)] + + +def test_lookup_variants_by_vrs_identifier_empty_when_none_match( + client, session, data_provider, data_files, setup_router_db +): + """The lookup is collection-shaped: no match is an empty list, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + # A well-formed GA4GH id (32-char digest) that no seeded allele carries. + response = client.get(f"/api/v1/variants/vrs/{quote_plus('ga4gh:VA.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA')}") + + assert response.status_code == 200 + assert response.json() == [] + + +def test_lookup_variants_by_vrs_identifier_empty_without_read_permission( + client, session, data_provider, data_files, setup_router_db +): + """Existence-hiding: an identifier that matches only a private score set is filtered to an empty list, + indistinguishable from a genuine no-match — so the response never reveals the private allele exists.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # The identifier matches a real allele (by vrs_digest); the permission filter drops it to an empty list. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True)], + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [] + + +# --- as_of temporal behavior ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("va_endpoint", ["study-result", "functional-statement", "pathogenicity-statement"]) +def test_va_endpoint_404_when_no_mapping_data_at_as_of( + client, session, data_provider, data_files, setup_router_db, va_endpoint +): + """A past instant with no live mapping yields the single-resource 404 — the statement did not exist then. + + ``as_of`` is threaded into ``variant_annotation_context``, so a far-past instant sees no live mapping + record and the derived VA statement cannot be constructed. This is the deliberate contrast with the + score-set *collection* endpoints: an empty collection is 200 + empty, but a single derived resource + that does not exist at the requested instant is a 404. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/{va_endpoint}", params={"as_of": past.isoformat()}) + + assert response.status_code == 404 + assert "no mapping data exists" in response.json()["detail"] + + +def test_variant_study_result_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The study-result route reconstructs its subject at ``as_of`` and echoes the instant on ``X-As-Of``. + + Study result needs only live mapping (no calibration), so it cleanly exercises the positive path: a + far-future instant still sees the freshly-seeded substrate as live and returns 200. Omitting ``as_of`` + reports ``current``. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result" + + # No as_of: the header reports the current standing. + current_response = client.get(url) + assert current_response.status_code == 200 + assert current_response.headers["X-As-Of"] == "current" + + # Far future: the seeded mapping is live, so the subject reconstructs and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + +def test_lookup_variants_by_vrs_identifier_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The VRS lookup filters its matches by ``as_of`` (the allele link must be live at that instant).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + url = f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}" + + # Far future: the allele link is live, so the identifier resolves and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert len(future_response.json()) == 1 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + # Far past: nothing was live yet, so no allele matches — an empty collection, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.json() == [] diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py index eb81dee7c..a6c47a44f 100644 --- a/tests/scripts/conftest.py +++ b/tests/scripts/conftest.py @@ -3,15 +3,21 @@ from datetime import date import pytest +from sqlalchemy import select from mavedb.models.acmg_classification import ACMGClassification -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.collection import Collection from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession from mavedb.models.target_gene import TargetGene @@ -280,14 +286,17 @@ def make_dump_score_set(session, sample_user, dump_experiment, dump_licenses, du Args: variant_scores: one score_data dict per variant. Keys become the score columns. count_columns: count column names; each variant gets a count_data value for every one. - mapped: attach a mapped variant to each variant. + mapped: give each variant a mapping — a live ``MappingRecord`` with an authoritative allele on + the allele substrate, plus a row in the frozen ``MappedVariant`` table, since the dump still + sources ``mapped-variants.json`` from the latter. current: whether those mappings are current. False models a fully superseded score set, which the dump must treat as having no annotations at all. - post_mapped: whether each mapping carries a post-mapped VRS allele. False leaves it NULL, which - is how a variant the mapper could not place is stored, and which the README documents as - yielding a null `annotation`. Note that the shared `TEST_MINIMAL_MAPPED_VARIANT` uses an - empty dict here, a shape production never stores and the annotation layer cannot parse. - published: sets published_date, which the dump's selection query requires, and clears private. + placeable: whether the mapper placed each variant. False gives it a `MappingRecord` with no + allele link at all, which is how an unplaceable variant is stored — an `Allele`'s identity + is its post-mapped VRS id, so one cannot exist without a post-mapped representation. Do not + reintroduce a "linked allele with a NULL `post_mapped`" shape; no write path produces it. + published: publishes the score set — stamps `published_date`, which the dump's selection query + requires, and clears `private`, which permission checks read. Publishing sets both. cc0: whether the score set carries the CC0 license the dump requires. """ counter = {"n": 0} @@ -298,7 +307,7 @@ def _make( count_columns=(), mapped=True, current=True, - post_mapped=False, + placeable=True, published=True, cc0=True, ): @@ -318,6 +327,10 @@ def _make( modified_by=sample_user, licence_id=CC0_LICENSE_ID if cc0 else OTHER_LICENSE_ID, published_date=date(2024, 1, 1) if published else None, + # `ScoreSet.private` defaults to True, and publishing clears it alongside stamping + # `published_date` (see the publish route). Setting only the date would build a score set that + # is published *and* private — a state production never creates, and one that reads as + # unreadable to any permission check while still satisfying the dump's selection query. private=not published, dataset_columns={"score_columns": score_columns, "count_columns": list(count_columns)}, target_genes=[ @@ -357,7 +370,7 @@ def _make( **{ **TEST_MINIMAL_MAPPED_VARIANT, "current": current, - "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped else None, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if placeable else None, }, variant_id=variant.id, clingen_allele_id=f"CA{counter['n']}{index}", @@ -365,6 +378,30 @@ def _make( ) session.commit() + # The annotation and CSV artifacts read the allele substrate, not the row above. An + # unplaceable variant gets the record and no allele, matching the `if post_mapped_allele:` + # guard in `worker/jobs/variant_processing/mapping.py`. + record = seed_mapping_record( + session, + variant, + assay_level="genomic", + alleles=[ + AlleleSpec( + digest=f"dump-allele-{counter['n']}-{index}", + level="genomic", + is_authoritative=True, + clingen_allele_id=f"CA{counter['n']}{index}", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ] + if placeable + else [], + ) + if not current: + # A fully superseded score set: the record is in history, but nothing is live. + record.retire(session) + session.commit() + session.refresh(score_set) return score_set @@ -373,20 +410,30 @@ def _make( @pytest.fixture def add_clinvar_control(session): - """Attach a ClinVar clinical control to a mapped variant, for a given `MM_YYYY` release.""" - - def _add(mapped_variant, *, db_version, significance="Pathogenic", review_status="criteria provided"): - mapped_variant.clinical_controls.append( - ClinicalControl( - db_identifier="183058", - gene_symbol="PTEN", - clinical_significance=significance, - clinical_review_status=review_status, - db_name="ClinVar", - db_version=db_version, - ) + """Attach a ClinVar clinical control to a variant's authoritative allele, for an `MM_YYYY` release.""" + + def _add(variant, *, db_version, significance="Pathogenic", review_status="criteria provided"): + """Link a control to *variant*'s authoritative allele, which is where the CSV layer reads it.""" + allele = session.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .where(MappingRecord.variant_id == variant.id) + .where(MappingRecordAllele.is_authoritative.is_(True)) + ).first() + assert allele is not None, f"variant {variant.urn} has no authoritative allele to link a control to" + + control = ClinvarControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, ) - session.add(mapped_variant) + session.add(control) + session.commit() + session.add(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id)) session.commit() return _add diff --git a/tests/scripts/conftest_optional.py b/tests/scripts/conftest_optional.py index c2216f6d5..65b16e8a7 100644 --- a/tests/scripts/conftest_optional.py +++ b/tests/scripts/conftest_optional.py @@ -12,6 +12,7 @@ from mavedb.models.acmg_classification import ACMGClassification from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.lib.mondo import get_generic_disease_term from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -65,6 +66,7 @@ def _make(score_set, *, private=False, research_use_only=False, title=None): primary=not (private or research_use_only), private=private, calibration_metadata={}, + disease_term=get_generic_disease_term(session), created_by_id=sample_user.id, modified_by_id=sample_user.id, ) diff --git a/tests/scripts/test_audit_reverse_translation.py b/tests/scripts/test_audit_reverse_translation.py new file mode 100644 index 000000000..82c7f136d --- /dev/null +++ b/tests/scripts/test_audit_reverse_translation.py @@ -0,0 +1,582 @@ +# ruff: noqa: E402 + +import sqlite3 +from datetime import datetime, timezone + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") +biocommons = pytest.importorskip("biocommons") +bioutils = pytest.importorskip("bioutils") +pytest.importorskip("ga4gh.vrs") + +from mavedb.lib.seqrepo import SequenceNotFoundError +from sqlalchemy import select + +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.scripts.audit_reverse_translation import ( + ERROR, + SQL_CHECKS, + ProjectionPair, + check_foreign_refget_aliases, + check_refgets_against_seqrepo, + collect_outcomes, + round_trip_pair, + run_sql_check, +) +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record + +CHECKS = {check.name: check for check in SQL_CHECKS} + + +def _post_mapped(refget: str) -> dict: + return { + "type": "Allele", + "location": {"type": "SequenceLocation", "sequenceReference": {"refgetAccession": refget}}, + } + + +def _score_set_with_variants(session, make_score_set, urn, n): + score_set = make_score_set() + score_set.urn = urn + session.commit() + + variants = [ + Variant(urn=f"{urn}#{i}", score_set_id=score_set.id, hgvs_nt=f"c.{i}A>G", data={}) for i in range(1, n + 1) + ] + session.add_all(variants) + session.commit() + return score_set, variants + + +def _healthy_record(session, variant, n): + """A measured cdna allele in its measured projection group, with the genomic twin and the protein apex.""" + measured = GroupSpec( + hgvs_c=f"NM_000001.1:c.{n}A>G", hgvs_g=f"NC_000001.11:g.{1000 + n}A>G", hgvs_p=f"NP_000001.1:p.Lys{n}Glu" + ) + return seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"c-{n}", + level="cdna", + is_authoritative=True, + hgvs_c=f"NM_000001.1:c.{n}A>G", + clingen_allele_id=f"CA{n}", + group=measured, + post_mapped=_post_mapped("SQ.cdna"), + ), + AlleleSpec( + digest=f"g-{n}", + level="genomic", + hgvs_g=f"NC_000001.11:g.{1000 + n}A>G", + clingen_allele_id=f"CA{n}", + group=measured, + post_mapped=_post_mapped("SQ.genome"), + ), + AlleleSpec( + digest=f"p-{n}", + level="protein", + hgvs_p=f"NP_000001.1:p.Lys{n}Glu", + group=measured, + post_mapped=_post_mapped("SQ.protein"), + ), + ], + ) + + +def _count(session, name, score_set_ids): + return run_sql_check(session, CHECKS[name], score_set_ids, samples=5) + + +@pytest.mark.integration +class TestSqlChecks: + def test_a_healthy_graph_passes_every_error_check(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000001-a-1", 2) + for n, variant in enumerate(variants, start=1): + _healthy_record(session, variant, n) + + failing = { + check.name: finding.count + for check in SQL_CHECKS + if check.severity == ERROR and (finding := run_sql_check(session, check, [score_set.id], 5)).count + } + + assert failing == {} + + def test_flags_a_measured_allele_left_outside_its_projection_group(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000002-a-1", 1) + candidate = GroupSpec(hgvs_c="NM_000002.2:c.5A>G", hgvs_g="NC_000002.12:g.50A>G") + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec(digest="measured", level="cdna", is_authoritative=True, hgvs_c="NM_000002.1:c.5A>G"), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_000002.2:c.5A>G", group=candidate), + AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000002.12:g.50A>G", group=candidate), + ], + ) + + finding = _count(session, "measured_allele_outside_projection_group", [score_set.id]) + + assert finding.count == 1 + assert finding.samples[0]["hgvs"] == "NM_000002.1:c.5A>G" + + def test_reports_a_measured_indel_outside_the_groups_as_info_not_error(self, session, make_score_set): + # A deletion straddling codons 261/262 changes the protein as deleting codon 262 does, but is a + # different DNA change, so RT's codon-aligned candidates do not include it. + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000011-a-1", 1) + deletion = { + "type": "Allele", + "location": {"type": "SequenceLocation", "start": 67611951, "end": 67611954}, + "state": {"type": "LiteralSequenceExpression", "sequence": ""}, + } + candidate = GroupSpec(hgvs_c="NM_006565.4:c.784_786del", hgvs_g="NC_000016.10:g.67611953_67611955del") + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="measured", + level="genomic", + is_authoritative=True, + hgvs_g="NC_000016.10:g.67611952_67611954del", + post_mapped=deletion, + ), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_006565.4:c.784_786del", group=candidate), + AlleleSpec( + digest="rt-g", level="genomic", hgvs_g="NC_000016.10:g.67611953_67611955del", group=candidate + ), + ], + ) + + assert _count(session, "measured_allele_outside_projection_group", [score_set.id]).count == 0 + assert _count(session, "measured_indel_outside_projection_group", [score_set.id]).count == 1 + + def test_reports_a_change_straddling_two_codons_as_info_not_error(self, session, make_score_set): + # c.135_137delinsTGC changes the third base of codon 45 and the second of codon 46; RT's candidates for + # p.Val46Ala sit inside codon 46 (g.95025144_95025146) only. + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000013-a-1", 1) + + def located(start, end, sequence): + return { + "type": "Allele", + "location": { + "type": "SequenceLocation", + "sequenceReference": {"refgetAccession": "SQ.genome"}, + "start": start, + "end": end, + }, + "state": {"type": "LiteralSequenceExpression", "sequence": sequence}, + } + + candidate = GroupSpec(hgvs_c="NM_152416.4:c.137T>C", hgvs_g="NC_000008.11:g.95025145T>C") + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="measured", + level="genomic", + is_authoritative=True, + hgvs_g="NC_000008.11:g.95025143_95025145delinsTGC", + post_mapped=located(95025142, 95025145, "TGC"), + ), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_152416.4:c.137T>C", group=candidate), + AlleleSpec( + digest="rt-g", + level="genomic", + hgvs_g="NC_000008.11:g.95025145T>C", + group=candidate, + post_mapped=located(95025144, 95025145, "C"), + ), + AlleleSpec( + digest="rt-g-2", + level="genomic", + hgvs_g="NC_000008.11:g.95025145_95025146delinsCA", + group=GroupSpec(hgvs_g="NC_000008.11:g.95025145_95025146delinsCA"), + post_mapped=located(95025144, 95025146, "CA"), + ), + ], + ) + + assert _count(session, "measured_allele_outside_projection_group", [score_set.id]).count == 0 + assert _count(session, "measured_allele_beyond_rt_codon", [score_set.id]).count == 1 + + def test_reports_a_reference_identical_allele_as_info_not_error(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000014-a-1", 1) + candidate = GroupSpec(hgvs_c="NM_001130145.3:c.519G>A", hgvs_g="NC_000011.10:g.102114338T>A") + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="measured", level="genomic", is_authoritative=True, hgvs_g="NC_000011.10:g.102114338T=" + ), + AlleleSpec(digest="rt-c", level="cdna", hgvs_c="NM_001130145.3:c.519G>A", group=candidate), + AlleleSpec(digest="rt-g", level="genomic", hgvs_g="NC_000011.10:g.102114338T>A", group=candidate), + ], + ) + + assert _count(session, "measured_allele_outside_projection_group", [score_set.id]).count == 0 + assert _count(session, "measured_reference_allele_outside_projection_group", [score_set.id]).count == 1 + + def test_flags_an_accession_whose_alleles_carry_two_refgets(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000003-a-1", 2) + for n, (variant, refget) in enumerate(zip(variants, ("SQ.ncbi", "SQ.cdot")), start=1): + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"twin-{n}", + level="cdna", + is_authoritative=True, + hgvs_c=f"NM_007294.3:c.{n}A>G", + post_mapped=_post_mapped(refget), + ) + ], + ) + + finding = _count(session, "accession_with_multiple_refgets", [score_set.id]) + + assert finding.count == 1 + assert finding.samples[0]["accession"] == "NM_007294.3" + assert sorted(finding.samples[0]["refgets"]) == ["SQ.cdot", "SQ.ncbi"] + + @pytest.mark.parametrize( + "hgvs, flagged", + [ + (("NP_000004.1:p.Lys1Glu", "NP_000004.1:p.Lys1Glu="), 1), + (("NP_000004.1:p.Lys1Glu", "NP_000004.2:p.Lys1Glu"), 0), + ], + ids=["same accession", "different accessions"], + ) + def test_flags_one_clingen_allele_id_only_on_twins_on_one_accession(self, session, make_score_set, hgvs, flagged): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000004-a-1", 2) + for n, (variant, expression) in enumerate(zip(variants, hgvs), start=1): + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"caid-{n}", + level="protein", + is_authoritative=True, + hgvs_p=expression, + clingen_allele_id="PA999", + ) + ], + ) + + assert _count(session, "clingen_allele_id_on_twin_alleles", [score_set.id]).count == flagged + + def test_reports_a_twin_on_a_retired_allele_as_info_not_error(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000010-a-1", 2) + records = [ + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"caid-{n}", + level="genomic", + is_authoritative=True, + hgvs_g=expression, + clingen_allele_id="CA999", + ) + ], + ) + for n, (variant, expression) in enumerate( + zip(variants, ("NC_000007.14:g.3043839CA=", "NC_000007.14:g.2947765_2947766delinsCA")), start=1 + ) + ] + retired_at = datetime.now(timezone.utc) + records[0].valid_to = retired_at + for link in session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.mapping_record_id == records[0].id) + ): + link.valid_to = retired_at + session.commit() + + assert _count(session, "clingen_allele_id_on_twin_alleles", [score_set.id]).count == 0 + assert _count(session, "clingen_allele_id_on_retired_twin", [score_set.id]).count == 1 + + def test_flags_a_candidate_projection_group_with_one_member(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000005-a-1", 1) + measured = GroupSpec(hgvs_c="NM_000005.1:c.1A>G", hgvs_g="NC_000005.10:g.10A>G") + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest="measured", + level="cdna", + is_authoritative=True, + hgvs_c="NM_000005.1:c.1A>G", + group=measured, + ), + AlleleSpec(digest="twin", level="genomic", hgvs_g="NC_000005.10:g.10A>G", group=measured), + AlleleSpec( + digest="lonely", + level="cdna", + hgvs_c="NM_000005.1:c.3A>G", + group=GroupSpec(hgvs_c="NM_000005.1:c.3A>G"), + ), + ], + ) + + finding = _count(session, "malformed_projection_group", [score_set.id]) + + assert finding.count == 1 + assert finding.samples[0]["levels"] == "cdna" + + def test_does_not_flag_a_measured_group_with_one_member(self, session, make_score_set): + """An intronic genomic change has no cdna allele, so its measured group holds the genomic allele alone.""" + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000015-a-1", 1) + seed_mapping_record( + session, + variant, + assay_level="genomic", + alleles=[ + AlleleSpec( + digest="intronic", + level="genomic", + is_authoritative=True, + hgvs_g="NC_000005.10:g.10A>G", + group=GroupSpec(hgvs_c="NM_000465.4:c.123+5A>G", hgvs_g="NC_000005.10:g.10A>G"), + ) + ], + ) + + assert _count(session, "malformed_projection_group", [score_set.id]).count == 0 + + def test_flags_a_level_that_disagrees_with_the_hgvs(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000006-a-1", 1) + seed_mapping_record( + session, + variant, + alleles=[AlleleSpec(digest="mislevel", level="cdna", is_authoritative=True, hgvs_g="NC_000006.12:g.1A>G")], + ) + + assert _count(session, "level_disagrees_with_hgvs", [score_set.id]).count == 1 + + def test_flags_a_live_link_on_a_retired_record(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000007-a-1", 1) + record = _healthy_record(session, variant, 7) + record.valid_to = datetime.now(timezone.utc) + session.commit() + + assert _count(session, "live_link_on_retired_record", [score_set.id]).count == 3 + + def test_flags_a_mapped_record_with_no_authoritative_allele(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000008-a-1", 1) + seed_mapping_record(session, variant, alleles=[]) + session.add( + AnnotationEvent( + annotation_type="vrs_mapping", + variant_id=variant.id, + score_set_id=score_set.id, + disposition="present", + reason="mapped", + ) + ) + session.commit() + + assert _count(session, "mapped_record_without_authoritative_link", [score_set.id]).count == 1 + + def test_does_not_flag_a_record_whose_mapping_failed(self, session, make_score_set): + score_set, (variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000009-a-1", 1) + seed_mapping_record(session, variant, alleles=[]) + session.add( + AnnotationEvent( + annotation_type="vrs_mapping", + variant_id=variant.id, + score_set_id=score_set.id, + disposition="failed", + reason="failed", + ) + ) + session.commit() + + assert _count(session, "mapped_record_without_authoritative_link", [score_set.id]).count == 0 + + def test_scoping_excludes_other_score_sets(self, session, make_score_set): + clean, (clean_variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000010-a-1", 1) + _, (broken_variant,) = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000011-a-1", 1) + _healthy_record(session, clean_variant, 10) + seed_mapping_record( + session, + broken_variant, + alleles=[ + AlleleSpec(digest="b-measured", level="cdna", is_authoritative=True, hgvs_c="NM_000011.1:c.1A>G"), + AlleleSpec( + digest="b-lonely", + level="cdna", + hgvs_c="NM_000011.1:c.3A>G", + group=GroupSpec(hgvs_c="NM_000011.1:c.3A>G"), + ), + ], + ) + + assert _count(session, "malformed_projection_group", [clean.id]).count == 0 + assert _count(session, "malformed_projection_group", None).count == 1 + + +@pytest.mark.integration +class TestRefgetsAgainstSeqRepo: + def test_counts_alleles_off_the_canonical_refget_and_lists_unresolved_accessions(self, session, make_score_set): + score_set, variants = _score_set_with_variants(session, make_score_set, "urn:mavedb:00000012-a-1", 3) + specs = [ + ("NM_000012.1:c.1A>G", "SQ.right"), + ("NM_000012.1:c.2A>G", "SQ.wrong"), + ("NM_999999.1:c.1A>G", "SQ.unknown"), + ] + for n, (variant, (hgvs, refget)) in enumerate(zip(variants, specs), start=1): + seed_mapping_record( + session, + variant, + alleles=[ + AlleleSpec( + digest=f"sr-{n}", + level="cdna", + is_authoritative=True, + hgvs_c=hgvs, + post_mapped=_post_mapped(refget), + ) + ], + ) + + def resolve(accession): + if accession == "NM_000012.1": + return "SQ.right" + raise SequenceNotFoundError(accession) + + mismatch, unresolved = check_refgets_against_seqrepo(session, resolve, [score_set.id], samples=5) + + assert mismatch.count == 1 + assert mismatch.samples == [ + {"accession": "NM_000012.1", "stored": "SQ.wrong", "seqrepo": "SQ.right", "alleles": 1} + ] + assert unresolved.count == 1 + assert unresolved.samples[0]["accession"] == "NM_999999.1" + + +def _pair(**overrides): + values = dict( + mapping_record_id=1, + projection_group=1, + hgvs_c="NM_1.1:c.1A>G", + cdna_digest="c", + hgvs_g="NC_1.11:g.100A>G", + genomic_digest="g", + protein_digests=["p"], + protein_hgvs=["NP_1.1:p.Lys1Glu"], + ) + return ProjectionPair(**{**values, **overrides}) + + +DIGESTS = {"NC_1.11:g.100A>G": "g", "NP_1.1:p.Lys1Glu": "p", "NC_1.11:g.200A>G": "g-other"} + + +class TestRoundTripPair: + def test_reports_matches_when_both_sides_identify_as_the_stored_alleles(self): + outcomes = round_trip_pair( + _pair(), lambda c: "NC_1.11:g.100A>G", lambda c: "NP_1.1:p.(Lys1Glu)", DIGESTS.__getitem__ + ) + + assert [outcome for outcome, _ in outcomes] == ["genomic: matches", "protein: matches"] + + def test_reports_a_genomic_projection_that_identifies_differently(self): + outcomes = round_trip_pair( + _pair(), lambda c: "NC_1.11:g.200A>G", lambda c: "NP_1.1:p.Lys1Glu", DIGESTS.__getitem__ + ) + + assert outcomes[0][0] == "GENOMIC: DISAGREES" + assert outcomes[0][1]["derived"] == "NC_1.11:g.200A>G" + + def test_reports_a_protein_consequence_that_identifies_differently(self): + outcomes = round_trip_pair( + _pair(protein_digests=["p-other"]), + lambda c: "NC_1.11:g.100A>G", + lambda c: "NP_1.1:p.Lys1Glu", + DIGESTS.__getitem__, + ) + + assert outcomes[1][0] == "PROTEIN: DISAGREES" + + def test_separates_translation_failures_from_disagreements(self): + def fail(_): + raise ValueError("intronic") + + outcomes = round_trip_pair(_pair(), fail, fail, DIGESTS.__getitem__) + + assert [outcome for outcome, _ in outcomes] == ["genomic: could not re-derive", "protein: could not re-derive"] + + @pytest.mark.parametrize("derived", ["NP_1.1:p.?", "NP_1.1:p.(Met1?)"]) + def test_reports_a_start_codon_change_as_unknown_effect_not_a_failure(self, derived): + outcomes = round_trip_pair(_pair(), lambda c: "NC_1.11:g.100A>G", lambda c: derived, DIGESTS.__getitem__) + + assert outcomes[1][0] == "protein: unknown effect (start codon)" + + def test_two_notations_for_an_unchanged_protein_match(self): + pair = _pair(protein_digests=["p-synonymous"], protein_hgvs=["NP_1.1:p.Thr328="]) + digests = {**DIGESTS, "NP_1.1:p.Ter330Ter": "p-stop"} + + outcomes = round_trip_pair( + pair, lambda c: "NC_1.11:g.100A>G", lambda c: "NP_1.1:p.Ter330Ter", digests.__getitem__ + ) + + assert outcomes[1][0] == "protein: matches (unchanged)" + + def test_reports_a_record_with_no_protein_apex(self): + outcomes = round_trip_pair( + _pair(protein_digests=[], protein_hgvs=[]), lambda c: "NC_1.11:g.100A>G", lambda c: "", DIGESTS.__getitem__ + ) + + assert outcomes[1][0] == "protein: no apex on record" + + +class TestCollectOutcomes: + def test_orders_score_sets_by_failure_share_and_tallies_failure_reasons(self): + outcomes = collect_outcomes( + [ + ("urn:a", "present", "translated", 90), + ("urn:a", "failed", "translation_error", 10), + ("urn:b", "present", "translated", 50), + ("urn:b", "failed", "translation_failed", 30), + ("urn:b", "failed", "translation_error", 20), + ] + ) + + assert [o.urn for o in outcomes] == ["urn:b", "urn:a"] + assert outcomes[0].failed_pct == 50.0 + assert outcomes[0].failure_reasons.most_common(1) == [("translation_failed", 30)] + assert outcomes[1].total == 100 + + +class TestForeignRefgetAliases: + def test_flags_a_ga4gh_alias_that_is_not_the_sequences_own_digest(self): + db = sqlite3.connect(":memory:") + db.execute( + "CREATE TABLE seqalias (seqalias_id INTEGER, seq_id TEXT, alias TEXT, namespace TEXT, added TEXT, is_current INTEGER)" + ) + db.executemany( + "INSERT INTO seqalias VALUES (?, ?, ?, ?, ?, ?)", + [ + (1, "j1QR", "GS_j1QR", "VMC", "2017-07-19", 1), + (2, "j1QR", "GS__KAVUS", "VMC", "2026-10-02", 1), + (3, "j1QR", "NP_001346945.1", "NCBI", "2018-08-21", 1), + (4, "JtEW", "GS_jqmY", "VMC", "2026-10-08", 0), + ], + ) + + finding = check_foreign_refget_aliases(db, samples=5) + + assert finding.count == 1 + assert finding.samples[0]["alias"] == "GS__KAVUS" diff --git a/tests/scripts/test_audit_scope.py b/tests/scripts/test_audit_scope.py new file mode 100644 index 000000000..b52f4bd09 --- /dev/null +++ b/tests/scripts/test_audit_scope.py @@ -0,0 +1,87 @@ +# ruff: noqa: E402 + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("asyncclick") +pytest.importorskip("psycopg2") + +import asyncclick as click + +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.variant import Variant +from mavedb.scripts.audit_scope import resolve_scope + + +def _score_set(session, make_score_set, urn, translated_at): + """A score set with two variants, each reverse translated at the given instants (None: never).""" + score_set = make_score_set() + score_set.urn = urn + session.commit() + + for i, at in enumerate(translated_at, start=1): + variant = Variant(urn=f"{urn}#{i}", score_set_id=score_set.id, hgvs_nt=f"c.{i}A>G", data={}) + session.add(variant) + session.flush() + if at is not None: + session.add( + AnnotationEvent( + annotation_type="cross_level_translation", + variant_id=variant.id, + score_set_id=score_set.id, + disposition="present", + reason="translated", + created_at=at, + ) + ) + session.commit() + return score_set + + +OLD = datetime(2026, 9, 15, tzinfo=timezone.utc) +NEW = datetime(2026, 10, 7, tzinfo=timezone.utc) +SINCE = datetime(2026, 10, 1) + + +@pytest.mark.integration +class TestResolveScope: + def test_returns_none_when_unscoped(self, session): + assert resolve_scope(session, (), None, None) is None + + def test_translated_since_keeps_only_score_sets_every_variant_of_which_rt_reached(self, session, make_score_set): + current = _score_set(session, make_score_set, "urn:mavedb:00000020-a-1", [NEW, NEW]) + _score_set(session, make_score_set, "urn:mavedb:00000021-a-1", [OLD, OLD]) + _score_set(session, make_score_set, "urn:mavedb:00000022-a-1", [NEW, OLD]) + _score_set(session, make_score_set, "urn:mavedb:00000023-a-1", [None, None]) + + assert resolve_scope(session, (), None, SINCE) == [current.id] + + def test_options_intersect(self, session, make_score_set, sample_user): + in_collection = _score_set(session, make_score_set, "urn:mavedb:00000024-a-1", [NEW]) + stale_in_collection = _score_set(session, make_score_set, "urn:mavedb:00000025-a-1", [OLD]) + _score_set(session, make_score_set, "urn:mavedb:00000026-a-1", [NEW]) + + collection = Collection(name="audit", urn="urn:mavedb:collection-audit", created_by_id=sample_user.id) + session.add(collection) + session.flush() + for position, score_set in enumerate((in_collection, stale_in_collection)): + session.add( + CollectionScoreSetAssociation(collection_id=collection.id, score_set_id=score_set.id, position=position) + ) + session.commit() + + assert sorted(resolve_scope(session, (), collection.urn, None)) == sorted( + [in_collection.id, stale_in_collection.id] + ) + assert resolve_scope(session, (), collection.urn, SINCE) == [in_collection.id] + + def test_rejects_an_unknown_collection_and_an_empty_scope(self, session, make_score_set): + _score_set(session, make_score_set, "urn:mavedb:00000027-a-1", [OLD]) + + with pytest.raises(click.BadParameter): + resolve_scope(session, (), "urn:mavedb:collection-missing", None) + with pytest.raises(click.UsageError): + resolve_scope(session, (), None, SINCE) diff --git a/tests/scripts/test_backfill_campaign.py b/tests/scripts/test_backfill_campaign.py new file mode 100644 index 000000000..85bdcb851 --- /dev/null +++ b/tests/scripts/test_backfill_campaign.py @@ -0,0 +1,359 @@ +# ruff: noqa: E402 +"""Tests for the backfill campaign map (coverage) and safe-to-drop gate (reconcile).""" + +import pytest + +pytest.importorskip("arq") + +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant +from mavedb.scripts.backfill_campaign import ( + STATE_ENRICHED, + STATE_NONE, + STATE_PARTIAL_RESHAPE, + STATE_RESHAPED, + STATE_RT, + ScoreSetCoverage, + classify_state, + compute_coverage, + compute_reconcile, + resolve_cohort_ids, + scan_legacy_readers, +) +from tests.helpers.constants import TEST_MINIMAL_MAPPED_VARIANT +from tests.helpers.util.annotation import AlleleSpec, GroupSpec, seed_mapping_record + + +# Pure: state classification + + +def _cov(**kw) -> ScoreSetCoverage: + base = dict(score_set_id=1, urn="urn:mavedb:x-1", n_variants=10, n_legacy_variants=10, n_reshaped_variants=10) + base.update(kw) + return ScoreSetCoverage(**base) + + +@pytest.mark.unit +class TestClassifyState: + def test_none_when_no_live_record(self): + assert classify_state(_cov(n_reshaped_variants=0), require_current_vep=False) == STATE_NONE + + def test_partial_reshape(self): + cov = _cov(n_legacy_variants=10, n_reshaped_variants=4) + assert classify_state(cov, require_current_vep=False) == STATE_PARTIAL_RESHAPE + + def test_reshaped_without_rt(self): + cov = _cov(n_nonauthoritative_alleles=0) + assert classify_state(cov, require_current_vep=False) == STATE_RESHAPED + + def test_rt_when_fanout_present_but_enrichment_incomplete(self): + cov = _cov(n_nonauthoritative_alleles=5, n_gnomad_alleles=1) # missing clinvar/clingen/vep + assert classify_state(cov, require_current_vep=False) == STATE_RT + + def test_enriched_requires_all_sources(self): + cov = _cov( + n_nonauthoritative_alleles=5, + n_gnomad_alleles=1, + n_clinvar_alleles=1, + n_clingen_alleles=1, + n_vep_alleles=1, + ) + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + + def test_current_vep_gate_downgrades_when_only_stale_vep_present(self): + cov = _cov( + n_nonauthoritative_alleles=5, + n_gnomad_alleles=1, + n_clinvar_alleles=1, + n_clingen_alleles=1, + n_vep_alleles=1, # some VEP, but... + n_vep_current_alleles=0, # ...none at the pinned current version + ) + assert classify_state(cov, require_current_vep=True) == STATE_RT + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + + def test_fully_annotated_without_fanout_is_enriched(self): + # Reverse translation can legitimately produce zero siblings (single-level score set). Such a set, + # once fully annotated, must reach 'enriched' rather than being stranded at 'reshaped'. VEP presence + # (VEP depends on RT) confirms RT ran even with no fan-out. + cov = _cov( + n_nonauthoritative_alleles=0, + n_gnomad_alleles=1, + n_clinvar_alleles=1, + n_clingen_alleles=1, + n_vep_alleles=1, + ) + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + + def test_vep_present_without_fanout_reads_as_rt_when_incomplete(self): + # VEP live but a source still missing: RT ran (VEP implies it), so this is 'rt', not 'reshaped'. + cov = _cov(n_nonauthoritative_alleles=0, n_vep_alleles=1) # missing gnomad/clinvar/clingen + assert classify_state(cov, require_current_vep=False) == STATE_RT + + def test_is_reshaped_needs_a_legacy_denominator(self): + # A score set with no legacy rows can't be "reshaped" against a zero denominator. + assert not _cov(n_legacy_variants=0, n_reshaped_variants=5).is_reshaped + + +# Pure: static legacy-reader scan + + +def _write(root, rel, text): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +@pytest.mark.unit +class TestScanLegacyReaders: + def test_flags_a_query_reader_outside_the_allowlist(self, tmp_path): + _write(tmp_path, "src/mavedb/routers/rogue.py", "rows = db.scalars(select(MappedVariant)).all()\n") + hits = scan_legacy_readers(tmp_path) + assert hits == ["src/mavedb/routers/rogue.py:1"] + + def test_flags_join_and_loader_idioms(self, tmp_path): + _write( + tmp_path, + "src/mavedb/lib/x.py", + "q = select(Variant).join(MappedVariant)\no = joinedload(Foo.bar, MappedVariant)\n", + ) + assert scan_legacy_readers(tmp_path) == ["src/mavedb/lib/x.py:1", "src/mavedb/lib/x.py:2"] + + def test_flags_association_table_query(self, tmp_path): + _write( + tmp_path, "src/mavedb/lib/y.py", "n = select(func.count()).select_from(gnomad_variants_mapped_variants)\n" + ) + assert scan_legacy_readers(tmp_path) == ["src/mavedb/lib/y.py:1"] + + def test_ignores_declarations_imports_aliases_and_prose(self, tmp_path): + _write( + tmp_path, + "src/mavedb/models/variant.py", + ( + "from mavedb.models.mapped_variant import MappedVariant\n" + 'mapped_variants: Mapped[list["MappedVariant"]] = relationship("MappedVariant")\n' + "Alias = Union[ScoreSet, MappedVariant, Variant]\n" + "# a comment naming MappedVariant\n" + '"""a docstring naming MappedVariant."""\n' + "cleanup = delete(MappedVariant)\n" # a writer, removed with the drop — not a reader + ), + ) + assert scan_legacy_readers(tmp_path) == [] + + def test_allowlisted_files_are_skipped(self, tmp_path): + _write( + tmp_path, + "src/mavedb/scripts/export_public_data.py", + "rows = db.scalars(select(MappedVariant)).all()\n", + ) + assert scan_legacy_readers(tmp_path) == [] + + +# DB-backed: coverage and reconcile + + +def _add_variants(session, score_set, n): + variants = [] + for i in range(n): + v = Variant(urn=f"{score_set.urn}#{i}", score_set_id=score_set.id, hgvs_nt=f"c.{i + 1}A>G", data={}) + session.add(v) + variants.append(v) + session.commit() + return variants + + +def _add_legacy_mapped_variant(session, variant, *, current=True): + session.add(MappedVariant(**{**TEST_MINIMAL_MAPPED_VARIANT, "current": current}, variant_id=variant.id)) + session.commit() + + +def _published(session, make_score_set, urn): + ss = make_score_set(published=True) + ss.urn = urn + session.commit() + return ss + + +@pytest.mark.integration +class TestComputeCoverage: + def test_not_reshaped_score_set_is_none(self, session, make_score_set): + ss = _published(session, make_score_set, "urn:mavedb:00000900-a-1") + for v in _add_variants(session, ss, 3): + _add_legacy_mapped_variant(session, v) + + cov = compute_coverage(session, [ss.id])[ss.id] + assert cov.n_variants == 3 + assert cov.n_legacy_variants == 3 + assert cov.n_reshaped_variants == 0 + assert classify_state(cov, require_current_vep=False) == STATE_NONE + + def test_reshaped_only_score_set(self, session, make_score_set): + ss = _published(session, make_score_set, "urn:mavedb:00000901-a-1") + for i, v in enumerate(_add_variants(session, ss, 2)): + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, + v, + assay_level="genomic", + alleles=[AlleleSpec(digest=f"resh-{i}", level="genomic", is_authoritative=True)], + ) + + cov = compute_coverage(session, [ss.id])[ss.id] + assert cov.n_reshaped_variants == 2 + assert cov.is_reshaped + assert not cov.has_rt + assert classify_state(cov, require_current_vep=False) == STATE_RESHAPED + + def test_rt_and_enriched_score_set(self, session, make_score_set): + ss = _published(session, make_score_set, "urn:mavedb:00000902-a-1") + gnomad = GnomADVariant( + db_name="gnomAD", + db_identifier="1-1-A-G", + db_version="4", + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + control = ClinvarControl( + db_identifier="1", + gene_symbol="X", + clinical_significance="Pathogenic", + clinical_review_status="ok", + db_name="ClinVar", + db_version="2026-01", + ) + session.add_all([gnomad, control]) + session.commit() + + v = _add_variants(session, ss, 1)[0] + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, + v, + assay_level="genomic", + alleles=[ + AlleleSpec(digest="auth", level="genomic", is_authoritative=True, clingen_allele_id="CA1"), + # RT fan-out sibling carrying every source annotation: + AlleleSpec( + digest="sib", + level="cdna", + is_authoritative=False, + group=GroupSpec(), + vep_consequence="missense_variant", + clinvar_control_ids=(control.id,), + gnomad_variant_ids=(gnomad.id,), + ), + ], + ) + + cov = compute_coverage(session, [ss.id])[ss.id] + assert cov.has_rt + assert cov.n_nonauthoritative_alleles == 1 + assert cov.has_gnomad and cov.has_clinvar and cov.has_clingen and cov.has_vep(require_current=False) + assert classify_state(cov, require_current_vep=False) == STATE_ENRICHED + # Pinning VEP to a version the seed did not write (seed writes "116") downgrades to rt. + cov_pinned = compute_coverage(session, [ss.id], vep_source_version="999")[ss.id] + assert cov_pinned.n_vep_current_alleles == 0 + assert classify_state(cov_pinned, require_current_vep=True) == STATE_RT + # And pinning to the version the seed wrote counts it current again. + cov_match = compute_coverage(session, [ss.id], vep_source_version="116")[ss.id] + assert cov_match.n_vep_current_alleles == 1 + + +@pytest.mark.integration +class TestComputeReconcile: + def test_not_safe_when_a_legacy_variant_is_unreshaped(self, session, make_score_set, tmp_path): + ss = _published(session, make_score_set, "urn:mavedb:00000903-a-1") + for v in _add_variants(session, ss, 2): + _add_legacy_mapped_variant(session, v) + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) + assert report.legacy_current_variants == 2 + assert report.missing_reshape_variants == 2 + assert not report.measured_parity_ok + assert not report.safe_to_drop + + def test_safe_when_every_legacy_variant_is_reshaped_and_readers_clear(self, session, make_score_set, tmp_path): + ss = _published(session, make_score_set, "urn:mavedb:00000904-a-1") + for i, v in enumerate(_add_variants(session, ss, 2)): + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, + v, + assay_level="genomic", + alleles=[AlleleSpec(digest=f"ok-{i}", level="genomic", is_authoritative=True)], + ) + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) # empty tmp repo -> no readers + assert report.missing_reshape_variants == 0 + assert report.measured_parity_ok + assert report.annotation_parity_ok + assert report.readers_clear + assert report.safe_to_drop + + def test_annotation_regression_blocks_the_drop(self, session, make_score_set, tmp_path): + # A variant the frozen tables annotate for ClinVar, reshaped but with NO live substrate ClinVar link + # (the protein force-close case), must block the drop until enrichment restores it. + ss = _published(session, make_score_set, "urn:mavedb:00000906-a-1") + control = ClinvarControl( + db_identifier="9", + gene_symbol="X", + clinical_significance="Pathogenic", + clinical_review_status="ok", + db_name="ClinVar", + db_version="2026-01", + ) + session.add(control) + session.commit() + v = _add_variants(session, ss, 1)[0] + # Legacy MappedVariant carrying a ClinVar association... + mv = MappedVariant(**{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, variant_id=v.id) + mv.clinical_controls = [control] + session.add(mv) + session.commit() + # ...reshaped to the measured level, but with no live ClinVar link on the substrate. + seed_mapping_record( + session, v, assay_level="protein", alleles=[AlleleSpec(digest="p", level="protein", is_authoritative=True)] + ) + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) + assert report.measured_parity_ok # measured level is complete... + clinvar = next(p for p in report.annotation_parity if p.source == "clinvar") + assert clinvar.legacy_variants == 1 + assert clinvar.regressed_variants == 1 # ...but ClinVar serving would regress. + assert not report.annotation_parity_ok + assert not report.safe_to_drop + + def test_unexpected_reader_blocks_even_with_full_parity(self, session, make_score_set, tmp_path): + ss = _published(session, make_score_set, "urn:mavedb:00000905-a-1") + v = _add_variants(session, ss, 1)[0] + _add_legacy_mapped_variant(session, v) + seed_mapping_record( + session, v, assay_level="genomic", alleles=[AlleleSpec(digest="ok", level="genomic", is_authoritative=True)] + ) + _write(tmp_path, "src/mavedb/routers/rogue.py", "select(MappedVariant)\n") + + report = compute_reconcile(session, [ss.id], repo_root=tmp_path) + assert report.measured_parity_ok + assert not report.readers_clear + assert not report.safe_to_drop + + +@pytest.mark.integration +class TestResolveCohortIds: + def test_published_only_includes_accession_based(self, session, make_score_set): + # The accession-based-skip pitfall: --published-only must not discriminate on target type. + seq = make_score_set(published=True, gene_names=("BRCA1",)) + acc = make_score_set(published=True, gene_names=(), accession_gene_names=("TP53",)) + private = make_score_set(published=False, gene_names=("EGFR",)) + seq.urn, acc.urn, private.urn = "urn:mavedb:00000910-a-1", "urn:mavedb:00000911-a-1", "urn:mavedb:00000912-a-1" + session.commit() + + ids = resolve_cohort_ids( + session, published_only=True, include_private=False, collection_urn=None, explicit_urns=None + ) + assert seq.id in ids + assert acc.id in ids + assert private.id not in ids diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py index 6ca5b459a..8d651d7b1 100644 --- a/tests/scripts/test_export_public_data.py +++ b/tests/scripts/test_export_public_data.py @@ -40,6 +40,7 @@ score_set_has_current_mappings, scores_csv, va_ndjson, + vrs_ndjson, write_public_dump, ) from mavedb.view_models.experiment_set import ExperimentSetPublicDump @@ -53,6 +54,10 @@ def _header(csv_text): return next(csv.reader(io.StringIO(csv_text))) +def _variants_of(session, score_set): + return session.query(Variant).filter(Variant.score_set_id == score_set.id).order_by(Variant.id).all() + + def _mapped_variants_of(session, score_set): return ( session.query(MappedVariant) @@ -214,6 +219,7 @@ def test_a_mapped_score_set_contributes_every_annotation_artifact( assert set(artifacts) == { f"csv/{base}.scores.csv", f"csv/{base}.annotations.csv", + f"vrs/{base}.vrs.ndjson", f"mapped/{base}.mapped-variants.json", f"va/{base}.va.ndjson", } @@ -225,6 +231,21 @@ def test_a_superseded_score_set_contributes_scores_alone(self, session, make_dum assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + def test_legacy_mapped_variants_artifact_switches_off( + self, session, make_dump_score_set, anonymous_principal, monkeypatch + ): + """The deprecated legacy artifact drops cleanly at a version boundary via one env flag, without + touching the substrate-sourced artifacts that replace it.""" + score_set = make_dump_score_set(mapped=True, current=True) + base = archive_path_base(score_set.urn) + + monkeypatch.setenv("EXPORT_LEGACY_MAPPED_VARIANTS", "0") + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert f"mapped/{base}.mapped-variants.json" not in artifacts + # The substrate replacement still ships. + assert f"vrs/{base}.vrs.ndjson" in artifacts + def test_count_columns_add_the_counts_file(self, session, make_dump_score_set, anonymous_principal): score_set = make_dump_score_set(mapped=False, count_columns=("c_0",)) base = archive_path_base(score_set.urn) @@ -234,12 +255,12 @@ def test_count_columns_add_the_counts_file(self, session, make_dump_score_set, a assert set(artifacts) == {f"csv/{base}.scores.csv", f"csv/{base}.counts.csv"} def test_every_path_sits_in_a_documented_directory(self, session, make_dump_score_set, anonymous_principal): - """README `Archive Structure` lists exactly these three directories.""" + """README `Archive Structure` lists exactly these directories.""" score_set = make_dump_score_set(count_columns=("c_0",)) artifacts = _artifacts(session, score_set, anonymous_principal) - assert {path.split("/")[0] for path in artifacts} == {"csv", "mapped", "va"} + assert {path.split("/")[0] for path in artifacts} == {"csv", "vrs", "mapped", "va"} def test_every_artifact_carries_a_row_per_variant(self, session, make_dump_score_set, anonymous_principal): """Truthiness alone would accept a header with no rows, which is the failure worth catching.""" @@ -256,7 +277,7 @@ def test_every_artifact_carries_a_row_per_variant(self, session, make_dump_score assert len(json.loads(content)) == 2, path def test_yields_each_artifact_without_holding_the_others(self, session, make_dump_score_set, anonymous_principal): - """A generator, so the caller writes and releases each payload rather than accumulating four. + """A generator, so the caller writes and releases each payload rather than accumulating them all. One score set's `va.ndjson` runs to tens of kilobytes per variant, so returning them together made peak memory the sum of a score set's artifacts instead of its largest one. @@ -268,7 +289,7 @@ def test_yields_each_artifact_without_holding_the_others(self, session, make_dum first_path, first_content = next(artifacts) assert first_path == f"csv/{archive_path_base(score_set.urn)}.scores.csv" assert first_content - assert sum(1 for _ in artifacts) == 4 # annotations, mapped, va, counts still unevaluated + assert sum(1 for _ in artifacts) == 5 # annotations, vrs, mapped, va, counts still unevaluated #################################################################################################### @@ -339,7 +360,7 @@ def test_carries_exactly_the_expected_groups( Both are legitimate changes to make. Neither should be possible without editing this list. """ score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) - add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + add_clinvar_control(_variants_of(session, score_set)[0], db_version="01_2024") calibration = make_dump_calibration(score_set) namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) @@ -358,9 +379,9 @@ def test_carries_every_ingested_clinvar_release( ): """README: this file carries every release MaveDB holds, not just the most recent.""" score_set = make_dump_score_set() - mapped_variant = _mapped_variants_of(session, score_set)[0] - add_clinvar_control(mapped_variant, db_version="01_2024") - add_clinvar_control(mapped_variant, db_version="06_2025") + variant = _variants_of(session, score_set)[0] + add_clinvar_control(variant, db_version="01_2024") + add_clinvar_control(variant, db_version="06_2025") namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) @@ -421,7 +442,7 @@ def test_a_clinvar_release_carries_its_release_in_the_column_name( self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer ): score_set = make_dump_score_set() - add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + add_clinvar_control(_variants_of(session, score_set)[0], db_version="01_2024") header = _header(annotations_csv(session, score_set, anonymous_viewer)) @@ -535,6 +556,52 @@ def test_is_a_json_array(self, session, make_dump_score_set): #################################################################################################### +def _vrs_records(session, score_set): + return [json.loads(line) for line in vrs_ndjson(session, score_set).splitlines()] + + +@pytest.mark.integration +class TestVrsNdjson: + """The GA4GH molecular objects, in one artifact rather than scattered across the archive.""" + + def test_emits_one_line_per_mapped_variant(self, session, make_dump_score_set): + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + content = vrs_ndjson(session, score_set) + + assert len(content.splitlines()) == 3 + + def test_every_line_is_newline_terminated(self, session, make_dump_score_set): + """A line-based consumer should need no special case for the last record.""" + content = vrs_ndjson(session, make_dump_score_set()) + + assert content.endswith("\n") + + def test_carries_the_vrs_pair_and_the_categorical_variant(self, session, make_dump_score_set): + record = _vrs_records(session, make_dump_score_set())[0] + + assert set(record) == {"variant_urn", "pre_mapped", "post_mapped", "categorical_variant"} + assert record["post_mapped"]["type"] == "Allele" + assert record["categorical_variant"]["type"] == "CategoricalVariant" + + def test_an_unplaceable_variant_is_omitted(self, session, make_dump_score_set): + """An unplaceable variant has a `MappingRecord` but no allele, and every field here is + allele-derived, so it contributes no line rather than a line of nulls.""" + assert vrs_ndjson(session, make_dump_score_set(placeable=False)) == "" + + def test_a_score_set_with_no_mappings_yields_nothing(self, session, make_dump_score_set): + assert vrs_ndjson(session, make_dump_score_set(mapped=False)) == "" + + def test_is_not_withheld_for_a_private_calibration(self, session, make_dump_score_set, make_dump_calibration): + """None of this is calibration-derived, so calibration visibility must not gate it.""" + score_set = make_dump_score_set() + make_dump_calibration(score_set, private=True) + + record = _vrs_records(session, score_set)[0] + + assert record["post_mapped"] is not None + + def _va_records(session, score_set, principal): return [json.loads(line) for line in va_ndjson(session, score_set, principal).splitlines()] @@ -559,28 +626,23 @@ def test_every_line_is_newline_terminated(self, session, make_dump_score_set, an assert "\n\n" not in content def test_every_line_is_an_envelope_with_both_fields(self, session, make_dump_score_set, anonymous_principal): - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": 1.0}, {"score": 2.0})) + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) records = _va_records(session, score_set, anonymous_principal) assert all(set(record) == {"variant_urn", "annotation"} for record in records) - def test_an_unannotatable_variant_still_gets_its_urn(self, session, make_dump_score_set, anonymous_principal): - """README: `annotation` is null for a current mapping with no post-mapped allele.""" - score_set = make_dump_score_set(post_mapped=False) - - records = _va_records(session, score_set, anonymous_principal) - - assert records - assert all(record["variant_urn"] for record in records) - assert all(record["annotation"] is None for record in records) + def test_an_unplaceable_variant_is_omitted(self, session, make_dump_score_set, anonymous_principal): + """An unplaceable variant carries no allele, so it is not an annotatable variant and + contributes no line.""" + assert va_ndjson(session, make_dump_score_set(placeable=False), anonymous_principal) == "" def test_is_empty_for_a_score_set_with_no_current_mappings(self, session, make_dump_score_set, anonymous_principal): assert va_ndjson(session, make_dump_score_set(mapped=False), anonymous_principal) == "" def test_a_post_mapped_variant_carries_an_annotation(self, session, make_dump_score_set, anonymous_principal): """The counterpart to the null case: a post-mapped allele is what makes a variant annotatable.""" - score_set = make_dump_score_set(post_mapped=True) + score_set = make_dump_score_set() records = _va_records(session, score_set, anonymous_principal) @@ -591,7 +653,7 @@ def test_a_clinical_calibration_raises_the_record_to_the_pathogenicity_layer( self, session, make_dump_score_set, make_dump_calibration, anonymous_principal ): """README `va/{urn}.va.ndjson`: the highest materialized layer, which a calibration supplies.""" - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) make_dump_calibration(score_set, research_use_only=False) annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] @@ -607,7 +669,7 @@ def test_a_research_use_only_calibration_does_not_raise_the_layer( The record falls back to the functional-impact layer rather than disappearing, so the variant is still reported — just without a pathogenicity statement built on evidence not cleared for it. """ - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) make_dump_calibration(score_set, research_use_only=True) annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] @@ -618,7 +680,7 @@ def test_a_private_calibration_does_not_raise_the_layer( self, session, make_dump_score_set, make_dump_calibration, anonymous_principal ): """A private calibration is withheld from every artifact, this one included.""" - score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) make_dump_calibration(score_set, private=True) annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] @@ -648,7 +710,7 @@ def test_keeps_a_calibration_named_visible(self, session, make_dump_score_set, m session.refresh(score_set) view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) - narrowed = public_experiment_set(view, {calibration.id}) + narrowed = public_experiment_set(view, {calibration.id}, set()) kept = narrowed.experiments[0].score_sets[0].score_calibrations assert [c.id for c in kept] == [calibration.id] @@ -659,7 +721,7 @@ def test_drops_a_calibration_not_named_visible(self, session, make_dump_score_se session.refresh(score_set) view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) - narrowed = public_experiment_set(view, set()) + narrowed = public_experiment_set(view, set(), set()) assert narrowed.experiments[0].score_sets[0].score_calibrations == [] @@ -673,7 +735,49 @@ def test_returns_none_when_no_experiment_has_a_score_set(self, experiment_set_vi } ) - assert public_experiment_set(emptied, set()) is None + assert public_experiment_set(emptied, set(), set()) is None + + def _superseded_view(self, session, score_set, successor, readable_superseding_urns): + score_set.superseding_score_set = successor + session.commit() + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).first()) + + narrowed = public_experiment_set(view, set(), readable_superseding_urns) + return next(ss for e in narrowed.experiments for ss in e.score_sets if ss.urn == str(score_set.urn)) + + def test_blanks_a_supersession_the_caller_may_not_read(self, session, make_dump_score_set): + """A published score set is usually superseded by a private in-progress replacement. + + `superseding_score_set` is inherited from `SavedScoreSet` and hydrated from the ORM relationship + during validation, so without narrowing it the archive names an unreleased score set's URN and + title in `main.json` — published to Zenodo, where it cannot be recalled. + """ + superseded = self._superseded_view(session, make_dump_score_set(), make_dump_score_set(published=False), set()) + + assert superseded.superseding_score_set is None + + def test_keeps_a_supersession_the_caller_may_read(self, session, make_dump_score_set): + """The blank withholds only what the caller cannot already see.""" + successor = make_dump_score_set() + superseded = self._superseded_view(session, make_dump_score_set(), successor, {str(successor.urn)}) + + assert superseded.superseding_score_set is not None + assert superseded.superseding_score_set.urn == str(successor.urn) + + def test_keeps_a_readable_supersession_whose_data_the_archive_does_not_carry(self, session, make_dump_score_set): + """Gated on READ, not on archive membership. + + A published successor under a non-CC0 license is public knowledge; its data living outside this + archive is no reason to hide that the score set it replaced is superseded. Suppressing the pointer + would leave a consumer treating a superseded score set as current, which is the worse failure. The + successor is deliberately absent from the archive here, and must still be named. + """ + successor = make_dump_score_set(cc0=False) + superseded = self._superseded_view(session, make_dump_score_set(), successor, {str(successor.urn)}) + + assert superseded.superseding_score_set is not None + assert superseded.superseding_score_set.urn == str(successor.urn) def test_does_not_mutate_the_orm_graph(self, session, make_dump_score_set, make_dump_calibration): """`ScoreSet.score_calibrations` cascades delete-orphan, so narrowing the ORM collection instead @@ -685,7 +789,7 @@ def test_does_not_mutate_the_orm_graph(self, session, make_dump_score_set, make_ session.refresh(score_set) view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) - public_experiment_set(view, set()) + public_experiment_set(view, set(), set()) session.commit() assert session.query(ScoreCalibration).filter(ScoreCalibration.id == calibration.id).one_or_none() @@ -754,6 +858,16 @@ def test_separates_the_two_within_one_experiment(self, session, make_dump_score_ #################################################################################################### +def _metadata_supersessions(metadata): + """Every score set's reported superseding URN, keyed by its own URN.""" + return { + score_set.urn: (score_set.superseding_score_set.urn if score_set.superseding_score_set else None) + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + } + + def _metadata_calibration_ids(metadata): return { calibration.id @@ -817,6 +931,69 @@ def test_carries_a_research_use_only_calibration( assert _metadata_calibration_ids(metadata) == {calibration_id} + @pytest.mark.parametrize( + "successor_published,successor_cc0,expect_named", + [ + (True, True, True), # a successor the archive also carries + (True, False, False), # published, but its data is not in the archive + (False, True, False), # an in-progress replacement + (False, False, False), + ], + ) + def test_supersession_is_named_only_when_the_archive_carries_the_successor( + self, session, make_dump_score_set, anonymous_principal, successor_published, successor_cc0, expect_named + ): + """What `main.json` says about supersession, and why. + + `published_experiment_sets` loads members through `lazyload(...).and_(published + CC0)`, and that + criteria propagates to the self-referential `superseding_score_set` relationship. So a successor + hydrates exactly when it satisfies the member filter itself — the relationship is fine, an + unfiltered query finds every one of these. + + Two consequences worth pinning: + + - A private, in-progress successor is never named, so the narrowing in `public_experiment_set` is + defence in depth against this path rather than a live gate. + - A *published* successor whose license keeps its data out of the archive is not named either. That + one is a real gap: the score set is public knowledge, and a consumer is left reading a superseded + score set as current. Closing it needs the successor fetched by a separate unfiltered query and + injected into the view, not merely left un-blanked. + """ + superseded = make_dump_score_set() + successor = make_dump_score_set(published=successor_published, cc0=successor_cc0) + superseded.superseding_score_set = successor + session.commit() + urn, successor_urn = superseded.urn, successor.urn + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_supersessions(metadata)[urn] == (successor_urn if expect_named else None) + + @pytest.mark.parametrize("meta_analysis_published", [False, True]) + def test_names_only_readable_meta_analyses( + self, session, make_dump_score_set, anonymous_principal, meta_analysis_published + ): + """Anyone may start a meta-analysis of a published score set, so a private one must never be named + in an archive that can't be recalled. The loader's published+CC0 filter withholds it, and + `public_experiment_set` narrows by READ as a second line.""" + analyzed = make_dump_score_set() + meta_analysis = make_dump_score_set(published=meta_analysis_published) + meta_analysis.meta_analyzes_score_sets = [analyzed] + session.commit() + urn, meta_analysis_urn = analyzed.urn, meta_analysis.urn + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + meta_analyzed_by = { + score_set.urn: score_set.meta_analyzed_by_score_set_urns + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + } + assert meta_analyzed_by[urn] == ([meta_analysis_urn] if meta_analysis_published else []) + def test_reports_the_urns_whose_artifacts_the_archive_carries( self, session, make_dump_score_set, anonymous_principal ): diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py index eae324466..b5251a7d4 100644 --- a/tests/scripts/test_run_score_set_pipelines.py +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -11,8 +11,8 @@ from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline from mavedb.scripts.run_score_set_pipelines import ( - CLUSTER_KEY_UNKNOWN, _IN_FLIGHT_STATUSES, + CLUSTER_KEY_UNKNOWN, cluster_cohort, cohort_filename, effective_pipeline_name, @@ -22,6 +22,7 @@ in_flight_pipelines, is_current, is_failure, + latest_pipeline_by_score_set, pipelines_by_score_set, plan_enqueue, render_cluster_table, @@ -287,6 +288,61 @@ def test_limit_caps_below_slots(self): ) assert sorted(self._decisions(plan).values()) == ["enqueue", "skip_cap", "skip_cap"] + def test_untried_entry_wins_a_slot_over_an_earlier_failed_entry(self): + """Cohort order (id 1 before id 2) must not decide this: id 1 previously failed, + id 2 has never run, and with only one slot the untried one has to get it — + otherwise a chronic failure at the front of the order retries forever while an + untried entry behind it never gets a turn.""" + plan = plan_enqueue( + self._cohort(2), + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1}), + slots=1, + limit=None, + ) + decisions = self._decisions(plan) + assert decisions[1] == "skip_cap" + assert decisions[2] == "enqueue" + + def test_failed_entry_gets_leftover_slots_after_untried_are_placed(self): + plan = plan_enqueue( + self._cohort(2), + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1}), + slots=2, + limit=None, + ) + assert set(self._decisions(plan).values()) == {"enqueue"} + + def test_limit_also_applies_across_the_two_priority_passes(self): + plan = plan_enqueue( + self._cohort(2), + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1}), + slots=2, + limit=1, + ) + decisions = self._decisions(plan) + assert decisions[2] == "enqueue" + assert decisions[1] == "skip_cap" + + def test_current_and_in_flight_still_take_precedence_over_failed_status(self): + cohort = self._cohort(2) + plan = plan_enqueue( + cohort, + in_flight_score_set_ids={1}, + current_score_set_ids={2}, + failed_score_set_ids=frozenset({1, 2}), + slots=5, + limit=None, + ) + decisions = self._decisions(plan) + assert decisions[1] == "skip_in_flight" + assert decisions[2] == "skip_current" + def test_slots_fill_one_cluster_before_moving_to_the_next(self): """Two slots against two clusters must both land on the same gene, otherwise each running pipeline warms a ClinGen cache the other never reads.""" @@ -295,6 +351,21 @@ def test_slots_fill_one_cluster_before_moving_to_the_next(self): enqueued = {key for _ss, key, decision in plan if decision == "enqueue"} assert len(enqueued) == 1 + def test_failed_retry_pass_still_fills_one_cluster_before_the_next(self): + """The retry pass is a subsequence of the same cluster-ordered cohort, so it + inherits cache-coherent fill without any special-casing.""" + cohort = self._cohort(4, {1: "BRCA1", 2: "BRCA1", 3: "TP53", 4: "TP53"}) + plan = plan_enqueue( + cohort, + in_flight_score_set_ids=set(), + current_score_set_ids=set(), + failed_score_set_ids=frozenset({1, 2, 3, 4}), + slots=2, + limit=None, + ) + enqueued = {key for _ss, key, decision in plan if decision == "enqueue"} + assert len(enqueued) == 1 + def test_unknown_cluster_yields_to_real_clusters(self): cohort = self._cohort(3, {3: "TP53"}) plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=1, limit=None) @@ -311,7 +382,11 @@ class TestResolveJobSubset: def test_caid_leaf_resolves_against_map_annotate_score_set(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"submit_score_set_mappings_to_car"})) - assert {j["key"] for j in subset} == {"map_variants_for_score_set", "submit_score_set_mappings_to_car"} + assert {j["key"] for j in subset} == { + "reverse_translate_variants_for_score_set", + "map_variants_for_score_set", + "submit_score_set_mappings_to_car", + } def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] @@ -319,33 +394,28 @@ def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): { "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } ) subset = resolve_job_subset(jobs, leaf) assert {j["key"] for j in subset} == { + "reverse_translate_variants_for_score_set", "map_variants_for_score_set", "submit_score_set_mappings_to_car", "warm_clingen_cache", "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } - # TODO(#772) - @pytest.mark.skip(reason="vep currently disabled") def test_vep_leaf_resolves_against_map_annotate_score_set(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) assert {j["key"] for j in subset} == { "map_variants_for_score_set", - "submit_score_set_mappings_to_car", + "reverse_translate_variants_for_score_set", "populate_vep_for_score_set", } @@ -357,14 +427,11 @@ def test_vep_leaf_resolves_against_map_annotate_score_set(self): { "link_gnomad_variants", "refresh_clinvar_controls", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", } ), - # TODO(#772) - # frozenset({"populate_vep_for_score_set"}), + frozenset({"populate_vep_for_score_set"}), ], ) def test_presets_against_annotate_score_set_exclude_mapping_job(self, leaf): @@ -377,8 +444,6 @@ def test_missing_leaf_key_raises(self): with pytest.raises(ValueError): resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) - # TODO(#772) - @pytest.mark.skip(reason="vep currently disabled") def test_preserves_base_pipeline_order(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) @@ -512,6 +577,29 @@ def test_phase_and_base_pipeline_tracked_independently(self, session, make_score assert score_set.id not in succeeded +@pytest.mark.integration +class TestLatestPipelineByScoreSet: + def test_score_set_with_no_pipeline_is_absent(self, session, make_score_set): + score_set = make_score_set() + + result = latest_pipeline_by_score_set(session, tracked_name="test_pipeline", score_set_ids=[score_set.id]) + assert score_set.id not in result + + def test_most_recently_created_pipeline_wins(self, session, make_score_set): + score_set = make_score_set() + older = _make_pipeline( + session, status=PipelineStatus.FAILED, created_at=datetime(2024, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=older.id, score_set_id=score_set.id) + newer = _make_pipeline( + session, status=PipelineStatus.SUCCEEDED, created_at=datetime(2024, 6, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=newer.id, score_set_id=score_set.id) + + result = latest_pipeline_by_score_set(session, tracked_name="test_pipeline", score_set_ids=[score_set.id]) + assert result[score_set.id].id == newer.id + + @pytest.mark.integration class TestResolveCohort: def test_published_only_filters_unpublished(self, session, make_score_set): diff --git a/tests/validation/test_identifier.py b/tests/validation/test_identifier.py index 005278307..9cbd16b0d 100644 --- a/tests/validation/test_identifier.py +++ b/tests/validation/test_identifier.py @@ -63,6 +63,10 @@ def test_ve_invalid_list(self): def test_passes_valid_id(self): validate_ensembl_identifier("ENSG00000143384") + def test_passes_valid_versioned_id(self): + validate_ensembl_identifier("ENSP00000369497.3") + validate_ensembl_list(["ENSG00000139618.15", "ENST00000380152.8"]) + class TestRefSeqValidators(TestCase): """ diff --git a/tests/view_models/test_allele_measurement.py b/tests/view_models/test_allele_measurement.py new file mode 100644 index 000000000..39ef233fa --- /dev/null +++ b/tests/view_models/test_allele_measurement.py @@ -0,0 +1,19 @@ +from mavedb.view_models.allele_measurement import AlleleMeasurement +from tests.helpers.constants import TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL + +MEASUREMENT = { + "variantUrn": "urn:mavedb:00000001-a-1#1", + "relationship": "direct", + "scoreSetUrn": "urn:mavedb:00000001-a-1", + "scoreSetTitle": "Score set", + "isCurrent": True, +} + + +def test_allele_measurement_classification_defaults_to_clinical(): + measurement = AlleleMeasurement.model_validate( + {**MEASUREMENT, "classification": TEST_SAVED_FUNCTIONAL_RANGE_ABNORMAL} + ) + + assert measurement.classification is not None + assert not measurement.classification_is_research_use_only diff --git a/tests/view_models/test_calibration_control.py b/tests/view_models/test_calibration_control.py new file mode 100644 index 000000000..587782846 --- /dev/null +++ b/tests/view_models/test_calibration_control.py @@ -0,0 +1,154 @@ +from copy import deepcopy +from datetime import date + +import pytest +from pydantic import ValidationError + +from mavedb.lib.mondo import MONDO_SYSTEM +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.view_models.calibration_control import ( + CalibrationControlCreate, + SavedCalibrationControl, +) +from mavedb.view_models.score_calibration import ScoreCalibration, ScoreCalibrationCreate +from tests.helpers.constants import ( + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_SAVED_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_USER, +) +from tests.helpers.util.common import dummy_attributed_object_from_dict + +TEST_CONTROL_VARIANT_URN = "urn:mavedb:00000001-a-1#1" + + +def _saved_user_like(): + """An attributed object standing in for a persisted User (audit field).""" + return dummy_attributed_object_from_dict({"username": TEST_USER["username"]}) + + +def _control_attributed_object(status=CalibrationControlStatus.pathogenic): + """An attributed object standing in for a persisted CalibrationControl ORM row.""" + return dummy_attributed_object_from_dict( + { + "id": 1, + "variant": dummy_attributed_object_from_dict({"urn": TEST_CONTROL_VARIANT_URN}), + "clinical_status": status, + "creation_date": date(2026, 9, 16), + "modification_date": date(2026, 9, 16), + "created_by": _saved_user_like(), + "modified_by": _saved_user_like(), + } + ) + + +############################################################################## +# CalibrationControl view models from dicts (request bodies) +############################################################################## + + +def test_calibration_control_create_from_dict(): + control = CalibrationControlCreate(**{"variantUrn": TEST_CONTROL_VARIANT_URN, "clinicalStatus": "pathogenic"}) + + assert control.variant_urn == TEST_CONTROL_VARIANT_URN + assert control.clinical_status is CalibrationControlStatus.pathogenic + assert control.model_dump(by_alias=True)["variantUrn"] == TEST_CONTROL_VARIANT_URN + + +@pytest.mark.parametrize("status", ["pathogenic", "benign"]) +def test_calibration_control_create_accepts_both_statuses(status): + control = CalibrationControlCreate(variant_urn=TEST_CONTROL_VARIANT_URN, clinical_status=status) + assert control.clinical_status is CalibrationControlStatus(status) + + +def test_calibration_control_create_rejects_unknown_status(): + with pytest.raises(ValidationError): + CalibrationControlCreate(variant_urn=TEST_CONTROL_VARIANT_URN, clinical_status="likely_pathogenic") + + +def test_calibration_control_create_requires_variant_urn(): + with pytest.raises(ValidationError): + CalibrationControlCreate(clinical_status="benign") + + +############################################################################## +# SavedCalibrationControl from attributed objects (ORM models) +############################################################################## + + +def test_saved_calibration_control_synthesizes_variant_urn_from_orm(): + saved = SavedCalibrationControl.model_validate(_control_attributed_object()) + + # variant_urn is not a column on the ORM row; it is derived from the variant relationship. + assert saved.variant_urn == TEST_CONTROL_VARIANT_URN + assert saved.clinical_status is CalibrationControlStatus.pathogenic + assert saved.id == 1 + assert saved.created_by.username == TEST_USER["username"] + assert saved.modified_by.username == TEST_USER["username"] + assert saved.record_type == "SavedCalibrationControl" + + dumped = saved.model_dump(by_alias=True) + assert dumped["variantUrn"] == TEST_CONTROL_VARIANT_URN + assert dumped["recordType"] == "SavedCalibrationControl" + + +############################################################################## +# ScoreCalibration view models extended with controls / disease / controls_not_phi +############################################################################## + + +def test_score_calibration_create_accepts_controls_disease_and_phi(): + payload = deepcopy(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + payload["disease"] = "MONDO:0015263" + payload["controls_not_phi"] = True + payload["controls"] = [ + {"variant_urn": TEST_CONTROL_VARIANT_URN, "clinical_status": "pathogenic"}, + {"variant_urn": "urn:mavedb:00000001-a-1#2", "clinical_status": "benign"}, + ] + + calibration = ScoreCalibrationCreate.model_validate(payload) + + assert calibration.disease == "MONDO:0015263" + assert calibration.controls_not_phi is True + assert len(calibration.controls) == 2 + assert calibration.controls[0].clinical_status is CalibrationControlStatus.pathogenic + assert calibration.controls[1].clinical_status is CalibrationControlStatus.benign + + +def test_saved_score_calibration_includes_controls_and_disease_concept(): + saved = deepcopy(TEST_SAVED_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + saved["disease_term"] = dummy_attributed_object_from_dict( + {"code": "MONDO:0015263", "system": MONDO_SYSTEM, "system_version": None, "label": "Brugada syndrome"} + ) + saved["controlsNotPhi"] = True + saved["controls"] = [ + { + "id": 1, + "variantUrn": TEST_CONTROL_VARIANT_URN, + "clinicalStatus": "pathogenic", + "creationDate": date(2026, 9, 16), + "modificationDate": date(2026, 9, 16), + "createdBy": {"orcidId": TEST_USER["username"]}, + "modifiedBy": {"orcidId": TEST_USER["username"]}, + } + ] + + calibration = ScoreCalibration.model_validate(dummy_attributed_object_from_dict(saved)) + + assert calibration.disease.conceptType == "Disease" + assert calibration.disease.primaryCoding.code.root == "MONDO:0015263" + assert calibration.disease.name == "Brugada syndrome" + assert calibration.controls_not_phi is True + assert len(calibration.controls) == 1 + assert calibration.controls[0].variant_urn == TEST_CONTROL_VARIANT_URN + assert calibration.controls[0].clinical_status is CalibrationControlStatus.pathogenic + # Placement is a computed column_property; an in-memory object never carries it, so it stays unset. + assert calibration.controls[0].functional_classification_id is None + + +def test_saved_score_calibration_controls_default_to_empty_list(): + saved = deepcopy(TEST_SAVED_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + saved.pop("controls", None) + + calibration = ScoreCalibration.model_validate(dummy_attributed_object_from_dict(saved)) + + assert calibration.controls == [] diff --git a/tests/worker/conftest_optional.py b/tests/worker/conftest_optional.py index 0f1d2e95f..ccd4208a0 100644 --- a/tests/worker/conftest_optional.py +++ b/tests/worker/conftest_optional.py @@ -4,6 +4,7 @@ import pytest from arq import ArqRedis from cdot.hgvs.dataproviders import RESTDataProvider +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from sqlalchemy.orm import Session from mavedb.worker.lib.managers.job_manager import JobManager @@ -52,6 +53,7 @@ def mock_worker_ctx(): """Create a mock worker context dictionary for testing.""" mock_redis = Mock(spec=ArqRedis) mock_hdp = Mock(spec=RESTDataProvider) + mock_seqrepo = Mock(spec=SeqRepoDataProxy) mock_pool = Mock(spec=ProcessPoolExecutor) # Don't mock the session itself to allow real DB interactions in tests @@ -60,5 +62,6 @@ def mock_worker_ctx(): return { "redis": mock_redis, "hdp": mock_hdp, + "seqrepo": mock_seqrepo, "pool": mock_pool, } diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index de835a56d..ac1cbe605 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1,9 +1,13 @@ import pytest +from sqlalchemy import select +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import DependencyType from mavedb.models.job_dependency import JobDependency from mavedb.models.job_run import JobRun from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -44,6 +48,38 @@ def map_variants_sample_params(with_populated_domain_data, sample_score_set, sam } +@pytest.fixture +def reverse_translate_variants_sample_params(with_populated_domain_data, sample_score_set): + """Provide sample parameters for reverse_translate_variants_for_score_set job.""" + + return { + "score_set_id": sample_score_set.id, + "correlation_id": "sample-reverse-translation-correlation-id", + } + + +@pytest.fixture +def sample_independent_reverse_translation_run(reverse_translate_variants_sample_params): + """Create a JobRun instance for the reverse_translate_variants_for_score_set job.""" + + return JobRun( + urn="test:reverse_translate_variants_for_score_set", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=reverse_translate_variants_sample_params, + ) + + +@pytest.fixture +def with_reverse_translation_run(session, sample_independent_reverse_translation_run): + """Add a reverse_translate_variants_for_score_set job run to the session.""" + + session.add(sample_independent_reverse_translation_run) + session.commit() + + @pytest.fixture def link_gnomad_variants_sample_params(with_populated_domain_data, sample_score_set): """Provide sample parameters for create_variants_for_score_set job.""" @@ -229,7 +265,7 @@ def setup_sample_variants_with_caid( # Add a variant and mapped variant to the database with a CAID variant = Variant( - urn="urn:variant:test-variant-with-caid", + urn=f"{score_set.urn}#1", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", @@ -249,6 +285,90 @@ def setup_sample_variants_with_caid( return variant, mapped_variant +@pytest.fixture +def setup_sample_alleles_with_caid(session, with_populated_domain_data, sample_link_gnomad_variants_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the gnomAD linkage job. The allele carries the CAID matched by the mocked Athena row, and the + allele is the authoritative measurement for the variant so the bandaid seam writes its VAS row. + """ + score_set = session.get(ScoreSet, sample_link_gnomad_variants_run.job_params["score_set_id"]) + + variant = Variant( + urn=f"{score_set.urn}#2", + score_set_id=score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, + ) + allele = Allele( + vrs_digest="test-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NM_000000.1:c.1A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) + session.commit() + + mapping_record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", + mapping_api_version="pytest.0.0", + ) + session.add(mapping_record) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_with_caid(session, setup_sample_alleles_with_caid): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying + the CAID the mocked gnomAD row matches. The authoritative allele is given a CAID with no gnomAD + match, so only the RT-derived allele can link. This isolates the requirement that gnomAD linkage + must cover the full allele set (authoritative + RT-derived), not just authoritative links — for + protein/coding score sets the genomic allele gnomAD knows is the RT-derived one. + """ + variant, authoritative_allele = setup_sample_alleles_with_caid + + # Authoritative allele's CAID intentionally has no gnomAD match, so it cannot be what links. + authoritative_allele.clingen_allele_id = "CA_NO_GNOMAD_MATCH" + session.add(authoritative_allele) + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000001.11:g.12345G>A", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele + + ## Uniprot Job Fixtures ## @@ -894,198 +1014,6 @@ def with_cleanup_job(session, sample_cleanup_job_run): session.commit() -## HGVS Population Job Fixtures ## - - -@pytest.fixture -def populate_hgvs_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_hgvs_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_hgvs_pipeline(): - """Create a pipeline instance for populate_hgvs_for_score_set job.""" - - return Pipeline( - urn="test:populate_hgvs_pipeline", - name="Populate HGVS Pipeline", - ) - - -@pytest.fixture -def sample_populate_hgvs_run(populate_hgvs_sample_params): - """Create a JobRun instance for populate_hgvs_for_score_set job.""" - - return JobRun( - urn="test:populate_hgvs_for_score_set", - job_type="populate_hgvs_for_score_set", - job_function="populate_hgvs_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_hgvs_sample_params, - ) - - -@pytest.fixture -def with_populate_hgvs_job(session, sample_populate_hgvs_run): - """Add a populate_hgvs_for_score_set job run to the session.""" - - session.add(sample_populate_hgvs_run) - session.commit() - - -@pytest.fixture -def with_populate_hgvs_pipeline(session, sample_populate_hgvs_pipeline): - """Add a populate_hgvs pipeline to the session.""" - - session.add(sample_populate_hgvs_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_hgvs_run_pipeline( - session, - with_populate_hgvs_job, - with_populate_hgvs_pipeline, - sample_populate_hgvs_run, - sample_populate_hgvs_pipeline, -): - """Provide a context with a populate_hgvs job run and pipeline.""" - - sample_populate_hgvs_run.pipeline_id = sample_populate_hgvs_pipeline.id - session.commit() - return sample_populate_hgvs_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_hgvs( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_hgvs_run -): - """Setup variants and mapped variants in the database for HGVS population testing.""" - score_set = session.get(ScoreSet, sample_populate_hgvs_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-hgvs", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - -# --- Variant Translation Fixtures --- - - -@pytest.fixture -def populate_variant_translations_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_variant_translations_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_variant_translations_pipeline(): - """Create a pipeline instance for populate_variant_translations_for_score_set job.""" - - return Pipeline( - urn="test:populate_variant_translations_pipeline", - name="Populate Variant Translations Pipeline", - ) - - -@pytest.fixture -def sample_populate_variant_translations_run(populate_variant_translations_sample_params): - """Create a JobRun instance for populate_variant_translations_for_score_set job.""" - - return JobRun( - urn="test:populate_variant_translations_for_score_set", - job_type="populate_variant_translations_for_score_set", - job_function="populate_variant_translations_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_variant_translations_sample_params, - ) - - -@pytest.fixture -def with_populate_variant_translations_job(session, sample_populate_variant_translations_run): - """Add a populate_variant_translations_for_score_set job run to the session.""" - - session.add(sample_populate_variant_translations_run) - session.commit() - - -@pytest.fixture -def with_populate_variant_translations_pipeline(session, sample_populate_variant_translations_pipeline): - """Add a populate_variant_translations pipeline to the session.""" - - session.add(sample_populate_variant_translations_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_variant_translations_run_pipeline( - session, - with_populate_variant_translations_job, - with_populate_variant_translations_pipeline, - sample_populate_variant_translations_run, - sample_populate_variant_translations_pipeline, -): - """Provide a context with a populate_variant_translations job run and pipeline.""" - - sample_populate_variant_translations_run.pipeline_id = sample_populate_variant_translations_pipeline.id - session.commit() - return sample_populate_variant_translations_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_translation( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_variant_translations_run -): - """Setup variants and mapped variants in the database for variant translation testing.""" - score_set = session.get(ScoreSet, sample_populate_variant_translations_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-translation", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - ## ClinGen Cache Warming Job Fixtures ## @@ -1223,59 +1151,128 @@ def sample_populate_vep_run_pipeline( @pytest.fixture -def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a variant and mapped variant with hgvs_assay_level for VEP testing.""" +def setup_sample_alleles_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the VEP consequence job. The allele carries an HGVS the job submits to VEP and is the + authoritative measurement for the variant, so the bandaid seam writes its per-variant VAS row. + + The HGVS lives on ``Allele.hgvs_c`` (a coding HGVS VEP resolves directly) — the new job reads its + submission string from the allele, not from ``MappedVariant.hgvs_assay_level``. + """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-variant-for-vep", + urn=f"{score_set.urn}#4", score_set_id=score_set.id, hgvs_nt="NM_007294.4:c.5A>G", hgvs_pro="NP_009225.1:p.Cys2Tyr", data={"hgvs_c": "NM_007294.4:c.5A>G", "hgvs_p": "NP_009225.1:p.Cys2Tyr"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-allele-vrs-digest", + level="cdna", + hgvs_c="NM_007294.4:c.5A>G", + post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, - hgvs_assay_level="NM_007294.4:c.5A>G", + score_set_id=variant.score_set_id, + assay_level="cdna", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele @pytest.fixture -def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a protein HGVS variant (NP_ accession) that VEP cannot resolve directly. +def setup_sample_protein_allele_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up an allele whose only HGVS is a protein HGVS (NP_ accession) that VEP cannot resolve + directly. - VEP's /vep/human/hgvs endpoint does not return results for protein HGVS strings like - NP_009225.1:p.Val1696His, so these must be recoded via Variant Recoder first. This fixture - exercises the recoder fallback path end-to-end. + VEP's /vep/human/hgvs endpoint returns no consequence for protein HGVS like + NP_009225.1:p.Val1696His, so the job must fall back to Variant Recoder. ``hgvs_g``/``hgvs_c`` are + left unset so the VEP payload resolves to ``hgvs_p``, exercising the recoder fallback path. """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:test-protein-variant-for-vep", + urn=f"{score_set.urn}#5", score_set_id=score_set.id, hgvs_pro="NP_009225.1:p.Val1696His", data={"hgvs_p": "NP_009225.1:p.Val1696His"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-protein-allele-vrs-digest", + level="protein", + hgvs_p="NP_009225.1:p.Val1696His", + post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, - hgvs_assay_level="NP_009225.1:p.Val1696His", + score_set_id=variant.score_set_id, + assay_level="protein", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_for_vep(session, setup_sample_alleles_for_vep): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying a + genomic HGVS of its own. Isolates the requirement that VEP linkage covers the full allele set + (authoritative + RT-derived), while the per-variant VAS fan-out stays authoritative-only. + """ + variant, authoritative_allele = setup_sample_alleles_for_vep + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-vep-allele-vrs-digest", + level="genomic", + hgvs_g="NC_000017.11:g.43124027T>C", + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000017.11:g.43124027T>C", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index f29842158..875eb635b 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -10,11 +10,12 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.clinvar import generate_clinvar_versions +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.worker.jobs.external_services.clinvar import _generate_clinvar_versions pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -43,7 +44,7 @@ async def test_refresh_clinvar_controls_e2e( arq_redis, arq_worker, standalone_worker_context, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): @@ -53,7 +54,7 @@ async def test_refresh_clinvar_controls_e2e( """ with ( patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=_E2E_VERSIONS, ), patch( @@ -65,27 +66,25 @@ async def test_refresh_clinvar_controls_e2e( await arq_worker.async_run() await arq_worker.run_check() - # The variant should be present in the 2025-01 archive. - clinical_controls = session.scalars(select(ClinicalControl)).all() + # Verify that clinical controls were added successfully — one row per ClinVar version + # that contains the variant, so there may be more than one. + clinical_controls = session.scalars(select(ClinvarControl)).all() assert len(clinical_controls) >= 1 assert all(cc.db_identifier == "3045425" for cc in clinical_controls) - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, - VariantAnnotationStatus.status == AnnotationStatus.SUCCESS, + # Verify that at least one present event was recorded for the allele. The job processes one + # event per ClinVar version; versions without the allele produce absent events, so filtering + # for present gives a stable assertion. Events are allele-keyed (no variant_id). + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, + AnnotationEvent.disposition == Disposition.PRESENT, ) ).all() - assert len(success_annotations) == 1 - - # Total annotations == versions processed. - all_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, - ) - ).all() - assert len(all_annotations) == len(_E2E_VERSIONS) + assert len(present_events) >= 1 + assert all(e.variant_id is None and e.allele_id is not None for e in present_events) + # Verify that the job run was completed successfully session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED @@ -101,7 +100,7 @@ async def test_latest_clinvar_archive_has_expected_schema(self): CLINVAR_FIELDS_TO_KEEP is present. Fails fast if ClinVar renames or removes a column before a deploy reaches production. """ - year, month = generate_clinvar_versions()[-1] + year, month = _generate_clinvar_versions()[-1] data = await fetch_clinvar_variant_data(month, year) assert len(data) > 0, "ClinVar archive returned no records" diff --git a/tests/worker/jobs/external_services/network/test_hgvs.py b/tests/worker/jobs/external_services/network/test_hgvs.py deleted file mode 100644 index 56f100e0b..000000000 --- a/tests/worker/jobs/external_services/network/test_hgvs.py +++ /dev/null @@ -1,54 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateHgvsForScoreSet: - """End-to-end test for HGVS population against the real ClinGen API.""" - - async def test_populate_hgvs_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Enqueue the HGVS population job, run the worker, and verify HGVS fields are populated.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - session.refresh(mapped_variant) - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.MAPPED_HGVS, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/network/test_uniprot.py b/tests/worker/jobs/external_services/network/test_uniprot.py index 506eb20f0..ff5d7f746 100644 --- a/tests/worker/jobs/external_services/network/test_uniprot.py +++ b/tests/worker/jobs/external_services/network/test_uniprot.py @@ -5,10 +5,14 @@ pytest.importorskip("arq") from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from tests.helpers.constants import TEST_REFSEQ_IDENTIFIER pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +# Mapping records the versioned transcript; ID mapping is submitted the unversioned form, which UniProt matches +# reliably (see id_mapping_query_accession). +VERSIONED_REFSEQ_ACCESSION = "NM_003345.5" +SUBMITTED_ACCESSION = "NM_003345" + @pytest.mark.asyncio @pytest.mark.integration @@ -31,7 +35,7 @@ async def test_uniprot_mapping_jobs_e2e( # Add an accession to the target gene's post mapped metadata target_gene = sample_score_set.target_genes[0] - target_gene.post_mapped_metadata = {"protein": {"sequence_accessions": [TEST_REFSEQ_IDENTIFIER]}} + target_gene.post_mapped_metadata = {"protein": {"sequence_accessions": [VERSIONED_REFSEQ_ACCESSION]}} session.commit() await arq_redis.enqueue_job( @@ -45,12 +49,12 @@ async def test_uniprot_mapping_jobs_e2e( submitted_jobs = sample_submit_uniprot_mapping_jobs_run_in_pipeline.metadata_["submitted_jobs"] assert "1" in submitted_jobs assert submitted_jobs["1"]["job_id"] is not None - assert submitted_jobs["1"]["accession"] == TEST_REFSEQ_IDENTIFIER + assert submitted_jobs["1"]["accession"] == SUBMITTED_ACCESSION # Verify that polling job params have been updated correctly session.refresh(sample_polling_job_for_submission_run_in_pipeline) assert sample_polling_job_for_submission_run_in_pipeline.job_params["mapping_jobs"] == { - "1": {"job_id": submitted_jobs["1"]["job_id"], "accession": TEST_REFSEQ_IDENTIFIER} + "1": {"job_id": submitted_jobs["1"]["job_id"], "accession": SUBMITTED_ACCESSION} } # Verify that the submission job was completed successfully diff --git a/tests/worker/jobs/external_services/network/test_variant_translations.py b/tests/worker/jobs/external_services/network/test_variant_translations.py deleted file mode 100644 index b45087dd9..000000000 --- a/tests/worker/jobs/external_services/network/test_variant_translations.py +++ /dev/null @@ -1,56 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateVariantTranslationsForScoreSet: - """End-to-end test for variant translation population against the real ClinGen API.""" - - async def test_populate_variant_translations_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Enqueue the variant translation job, run the worker, and verify translations are created.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VARIANT_TRANSLATION, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/network/test_vep.py b/tests/worker/jobs/external_services/network/test_vep.py index 61071fc42..52ed791d2 100644 --- a/tests/worker/jobs/external_services/network/test_vep.py +++ b/tests/worker/jobs/external_services/network/test_vep.py @@ -6,9 +6,12 @@ from sqlalchemy import select +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.enums.vep import VepConsequenceSource +from mavedb.models.vep_allele_consequence import VepAlleleConsequence pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -27,10 +30,10 @@ async def test_populate_vep_e2e( arq_worker, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Enqueue the VEP job, run the worker, and verify consequence and annotation are populated.""" - _, mapped_variant = setup_sample_variants_for_vep + """Enqueue the VEP job, run the worker, and verify the allele's consequence and annotation.""" + variant, allele = setup_sample_alleles_for_vep await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() @@ -42,55 +45,24 @@ async def test_populate_vep_e2e( session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + live = session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele.id, + VepAlleleConsequence.current, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS - - async def test_populate_vep_e2e_with_recoder_path( - self, - session, - arq_redis, - arq_worker, - sample_populate_vep_run_pipeline, - sample_populate_vep_pipeline, - setup_sample_protein_variant_for_vep, - ): - """VEP job uses Variant Recoder for a protein HGVS (NP_ accession) that VEP cannot resolve directly. - - NP_009225.1:p.Val1696His is a BRCA1 protein variant that VEP's /vep/human/hgvs endpoint - does not return a consequence for. The job must fall back to Variant Recoder, recode it - to a genomic HGVS, and then re-query VEP with the recoded string. - """ - _, mapped_variant = setup_sample_protein_variant_for_vep - - await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_vep_run_pipeline) - assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_vep_pipeline) - assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + assert live.functional_consequence is not None + assert live.access_date is not None + # The coding allele's consequence is read from its own transcript (#772), not from VEP's + # cross-transcript headline. + assert live.consequence_source == VepConsequenceSource.transcript + assert live.matched_transcript is not None + + # Events are allele-keyed; the variant resolves its status through the live link. + event = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.allele_id == allele.id, + AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert event.disposition == Disposition.PRESENT diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 616263dfa..d89b35aa2 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -5,16 +5,22 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from copy import deepcopy from unittest.mock import patch from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clingen import ( submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -23,6 +29,16 @@ from tests.helpers.constants import TEST_CLINGEN_LDH_LINKING_RESPONSE_BAD_REQUEST from tests.helpers.util.setup.worker import create_mappings_in_score_set + +def _car_response(caid, allele): + """A CAR registration response for ``allele``, listing its submitted HGVS as CAR does.""" + return { + "@id": caid, + "type": "nucleotide", + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(allele.post_mapped)]}], + } + + pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -51,9 +67,9 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_mappings( self, @@ -76,9 +92,9 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( self, @@ -101,9 +117,9 @@ async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_registered_alleles( self, @@ -147,18 +163,18 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 variants dedup to 1 allele → one allele-keyed event, failed (no CAR response). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_all_car_errors( self, @@ -184,18 +200,18 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( dummy_variant_mapping_job_run, ) - # Build error responses for every submitted HGVS — CAR explicitly rejected all of them - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele (same VRS digest). Build an error response for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { "errorType": "InvalidHGVS", - "hgvs": get_hgvs_from_post_mapped(mv.post_mapped) or "", + "hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", "message": "Invalid HGVS expression.", "description": "", - "inputLine": get_hgvs_from_post_mapped(mv.post_mapped) or "", - "position": str(i), + "inputLine": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", + "position": "0", } - for i, mv in enumerate(mapped_variants) ] with ( @@ -215,20 +231,109 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 1 allele, rejected by CAR → one failed event (reason=service_rejected). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "service_rejected" - async def test_submit_score_set_mappings_to_car_repeated_hgvs( + async def test_submit_score_set_mappings_to_car_event_per_allele( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Each allele — authoritative and derived — gets exactly one allele-keyed event; there is no + per-variant fan-out. Both alleles are registered with a CAID.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # The sample's 4 variants dedup to one authoritative allele. + authoritative_allele = session.scalars(select(Allele)).one() + + # Attach a second, derived (is_authoritative=False) allele to one variant's current mapping + # record. It shares the authoritative allele's HGVS (different vrs_digest) so it is submitted + # under the same line — the realistic shape is a different level, but identical HGVS is enough + # to exercise the multi-allele-per-variant fan-out. + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + derived_allele = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-derived", + level=authoritative_allele.level, + post_mapped={**deepcopy(authoritative_allele.post_mapped), "id": "derived-allele-id"}, + ) + session.add(derived_allele) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + score_set_id=a_link.score_set_id, + allele_id=derived_allele.id, + is_authoritative=False, + ) + ) + session.flush() + + def fake_dispatch(hgvs_list): + return [ + {"@id": f"CA{idx}", "type": "nucleotide", "genomicAlleles": [{"hgvs": [hgvs]}]} + for idx, hgvs in enumerate(hgvs_list) + ] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # One event per allele (2 alleles → 2 events), each keyed on its allele, not fanned per-variant. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 2 + assert all(e.variant_id is None for e in events) + assert {e.allele_id for e in events} == {authoritative_allele.id, derived_allele.id} + assert all(e.disposition == "present" for e in events) + + # Both alleles registered: registration breadth covers the derived allele too. + session.refresh(authoritative_allele) + session.refresh(derived_allele) + assert authoritative_allele.clingen_allele_id is not None + assert derived_allele.clingen_allele_id is not None + + async def test_submit_score_set_mappings_to_car_response_count_mismatch( self, mock_worker_ctx, session, @@ -252,14 +357,14 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles with repeated HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele → 1 submitted HGVS, but CAR returns 2 results. The count + # violates the one-result-per-input contract, so positional alignment can't be trusted + # and the whole batch must be rejected without writing any CAID. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ - { - "@id": "CA_DUPLICATE", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mapped_variants[0].post_mapped)}], - } + {"@id": "CA111111", "type": "nucleotide", "genomicAlleles": []}, + {"@id": "CA222222", "type": "nucleotide", "genomicAlleles": []}, ] with ( @@ -267,10 +372,62 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", return_value=registered_alleles_mock, ), - # Patch get_hgvs_from_post_mapped to return the same HGVS for all variants + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.FAILED + + # No CAID written — neither of the returned values is trusted. + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "api_error" + + async def test_submit_score_set_mappings_to_car_malformed_response( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # CAR returns a response that is neither an error (no "errorType") nor a registration + # (no "@id"). This must be surfaced as a rejection, not crash the loop with a KeyError. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [{"type": "nucleotide", "genomicAlleles": []}] + + with ( patch( - "mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", - return_value=get_hgvs_from_post_mapped(mapped_variants[0].post_mapped), + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, ), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), @@ -282,24 +439,19 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( ) assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == "CA_DUPLICATE" + assert result.status == JobStatus.FAILED - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # No CAID assigned; the one allele's event is failed (reason=malformed_response). + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "malformed_response" - async def test_submit_score_set_mappings_to_car_partial_failure( + async def test_submit_score_set_mappings_to_car_repeated_hgvs( self, mock_worker_ctx, session, @@ -312,7 +464,6 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures (some matched, some not) result in a succeeded outcome with failure annotations.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -324,16 +475,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Get mapped variants; return a CAR response that only matches the first variant - mapped_variants = session.scalars(select(MappedVariant)).all() - assert len(mapped_variants) == 4 - - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) + # 4 variants share 1 allele; CAR returns 1 response for the 1 submitted HGVS + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": "CA_DUPLICATE", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -353,33 +502,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 - # Verify only the first variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" + # 1 allele received the CAID + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 1 + assert alleles[0].clingen_allele_id == "CA_DUPLICATE" - # Verify annotation statuses: 1 success, 3 failed - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", - ) - ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", - ) + # 1 allele → one present event. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None - async def test_submit_score_set_mappings_to_car_hgvs_not_found( + async def test_submit_score_set_mappings_to_car_partial_failure( self, mock_worker_ctx, session, @@ -392,6 +530,7 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -403,21 +542,15 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant) - .join(Variant) - .where(Variant.score_set_id == submit_score_set_mappings_to_car_sample_job_run.job_params["score_set_id"]) - ).all() + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - # Patch ClinGenAlleleRegistryService to return registered alleles registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } - for mv in mapped_variants ] with ( @@ -425,8 +558,6 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", return_value=registered_alleles_mock, ), - # Patch get_hgvs_from_post_mapped to not find any HGVS in registered alleles - patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), ): @@ -436,21 +567,76 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), ) + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 + + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" + + # All 4 variant annotations succeeded + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", + ) + ).all() + assert len(present_events) == 1 + + async def test_submit_score_set_mappings_to_car_hgvs_not_found( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # Patch get_hgvs_from_post_mapped to return None for all alleles + with ( + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # Verify annotation statuses were rendered as failed — 4 variants, all failed + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "not_applicable" + assert event.reason == "no_hgvs" async def test_submit_score_set_mappings_to_car_propagates_exception( self, @@ -519,20 +705,16 @@ async def test_submit_score_set_mappings_to_car_success( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant).join(Variant).where(Variant.score_set_id == sample_score_set.id) - ).all() - assert len(mapped_variants) == 4 + # All 4 variants share 1 allele (same VRS digest from construct_mock_mapping_output) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - # Patch ClinGenAlleleRegistryService to return registered alleles registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } - for mv in mapped_variants ] with ( @@ -552,20 +734,520 @@ async def test_submit_score_set_mappings_to_car_success( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None + + async def test_submit_score_set_mappings_to_car_preexisting( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Already-registered allele is re-annotated as preexisting, not re-submitted.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # Pre-set the CAID on the allele to simulate prior registration + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_PRIOR" + session.flush() + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + ) as mock_dispatch, + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # CAR should NOT be called since the allele is already registered + mock_dispatch.assert_not_called() + assert result.status == JobStatus.SUCCEEDED + assert result.data["already_registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 0 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "preexisting" + assert event.event_metadata["clingen_allele_id"] == "CA_PRIOR" + + async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Force re-registration that returns the same CAID is a success with registration_source=reconfirmed.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_CONFIRMED" + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + registered_alleles_mock = [ + { + "@id": "CA_CONFIRMED", + "type": "nucleotide", + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(allele.post_mapped)]}], + } + ] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 1 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "reconfirmed" + + async def test_submit_score_set_mappings_to_car_force_reregister_replaces_a_caid_car_assigns_differently( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Under force, CAR's CAID replaces a different stored one, and the links the old one keyed are retired.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_BACKFILLED" + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version="v4.1", + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + clinvar_control = ClinvarControl( + gene_symbol="BRCA1", + clinical_significance="Pathogenic", + clinical_review_status="reviewed by expert panel", + db_name="ClinVar", + db_identifier="12345", + db_version="01_2026", + ) + session.add_all([gnomad_variant, clinvar_control]) + session.flush() + session.add_all( + [ + GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id), + ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=clinvar_control.id), + ] + ) + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=[_car_response("CA_FROM_CAR", allele)], + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + session.refresh(allele) + assert allele.clingen_allele_id == "CA_FROM_CAR" + + live_gnomad = session.scalars( + select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id, GnomadAlleleLink.current) + ).all() + live_clinvar = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id, ClinvarAlleleLink.current) + ).all() + assert live_gnomad == [] and live_clinvar == [] + + corrected = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", AnnotationEvent.reason == "corrected" + ) + ).one() + assert corrected.disposition == "present" + assert corrected.event_metadata["replaced_caid"] == "CA_BACKFILLED" + assert corrected.event_metadata["clingen_allele_id"] == "CA_FROM_CAR" + assert corrected.event_metadata["retired_gnomad_links"] == 1 + assert corrected.event_metadata["retired_clinvar_links"] == 1 + + async def test_submit_score_set_mappings_to_car_links_a_caid_whose_response_does_not_list_the_submitted_hgvs( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """CAR lists its own normalized spelling, so a positionally matched CAID is linked and the gap recorded.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + allele = session.scalars(select(Allele)).first() + submitted_hgvs = get_hgvs_from_post_mapped(allele.post_mapped) + normalized = { + "@id": "CA_NORMALIZED", + "type": "nucleotide", + "genomicAlleles": [{"hgvs": ["NC_000001.11:g.1A>T"]}], + } + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=[normalized], + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["unlisted_hgvs_allele_count"] == 1 + session.refresh(allele) + assert allele.clingen_allele_id == "CA_NORMALIZED" + + event = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).one() + assert (event.disposition, event.reason) == ("present", "created") + assert event.event_metadata["submitted_hgvs_unlisted"] is True + assert event.event_metadata["submitted_hgvs"] == submitted_hgvs + + async def test_submit_score_set_mappings_to_car_force_reregister_reconfirms_a_caid_whose_response_does_not_list_the_submitted_hgvs( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A forced re-registration verifies a stored CAID even when CAR spells the allele differently.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_STORED" + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + normalized = {"@id": "CA_STORED", "type": "nucleotide", "genomicAlleles": [{"hgvs": ["NC_000001.11:g.1A>T"]}]} + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=[normalized], + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + event = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).one() + assert (event.disposition, event.reason) == ("present", "reconfirmed") + assert event.event_metadata["submitted_hgvs_unlisted"] is True + + async def _add_derived_alleles_with_distinct_hgvs(self, session, count): + """Attach ``count`` extra alleles to the score set, each carrying a distinct HGVS so the CAR + job produces ``count + 1`` distinct submission lines (one per HGVS). Returns every allele.""" + authoritative_allele = session.scalars(select(Allele)).one() + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + + alleles = [authoritative_allele] + for i in range(count): + post_mapped = deepcopy(authoritative_allele.post_mapped) + post_mapped["expressions"][0]["value"] = f"NC_000001.11:g.{1000 + i}A>T" + derived = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-d{i}", + level=authoritative_allele.level, + post_mapped=post_mapped, + ) + session.add(derived) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + score_set_id=a_link.score_set_id, + allele_id=derived.id, + is_authoritative=False, + ) + ) + alleles.append(derived) + + session.flush() + return alleles + + async def test_submit_score_set_mappings_to_car_batches_submissions( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A large allele set is dispatched in fixed-size chunks rather than one PUT: 4 distinct HGVS + with a batch size of 2 yields two dispatch calls, and every allele is still registered.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + alleles = await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + dispatched_batches = [] + + def fake_dispatch(hgvs_list): + dispatched_batches.append(list(hgvs_list)) + return [ + {"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": [{"hgvs": [hgvs]}]} for hgvs in hgvs_list + ] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # 4 distinct HGVS at batch size 2 → two dispatch calls, neither exceeding the batch size, and + # together covering every submitted line exactly once. + assert len(dispatched_batches) == 2 + assert all(len(batch) <= 2 for batch in dispatched_batches) + assert sum(len(batch) for batch in dispatched_batches) == 4 + + # Every allele across both batches is registered. + for allele in alleles: + session.refresh(allele) + assert allele.clingen_allele_id is not None + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert all(event.disposition == "present" for event in events) + + async def test_submit_score_set_mappings_to_car_batch_failure_isolated( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A failed batch fails only its own alleles: the first chunk returns nothing (request failure) + while the second registers normally. Per-batch reconciliation keeps the two independent.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + call_count = {"n": 0} + + def fake_dispatch(hgvs_list): + call_count["n"] += 1 + # First batch mimics a request failure (dispatch_submissions returns [] on error); the rest + # register normally. + if call_count["n"] == 1: + return [] + return [ + {"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": [{"hgvs": [hgvs]}]} for hgvs in hgvs_list + ] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # The good batch links alleles, so the job overall succeeds despite the failed batch. + assert result.status == JobStatus.SUCCEEDED + assert call_count["n"] == 2 + + # Exactly the second batch's two alleles are registered; the first batch's two are not. + registered = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(registered) == 2 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert sum(event.disposition == "present" for event in events) == 2 + failed = [event for event in events if event.disposition == "failed"] + assert len(failed) == 2 + assert all(event.reason == "api_error" for event in failed) @pytest.mark.integration @@ -597,15 +1279,14 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } - for mv in mapped_variants ] with ( @@ -623,19 +1304,18 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -666,15 +1346,14 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } - for mv in mapped_variants ] with ( @@ -692,19 +1371,18 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run_in_pipeline) @@ -738,13 +1416,13 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -777,13 +1455,13 @@ async def test_submit_score_set_mappings_to_car_no_submission_endpoint( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -808,13 +1486,13 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -862,15 +1540,15 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # Verify annotation statuses were rendered as failed — 4 variants, all failed + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -903,9 +1581,6 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( # Patch ClinGenAlleleRegistryService to return only errors with no linked alleles registered_alleles_mock = [ {"errorType": "InvalidHGVS", "hgvs": "test"}, - {"errorType": "InvalidHGVS", "hgvs": "test2"}, - {"errorType": "InvalidHGVS", "hgvs": "test3"}, - {"errorType": "InvalidHGVS", "hgvs": "test4"}, ] with ( @@ -925,15 +1600,15 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # Verify annotation statuses were rendered as failed — 4 variants, all failed + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -952,7 +1627,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures result in SUCCEEDED status with per-variant failure annotations committed.""" + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed, job SUCCEEDED.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -964,14 +1639,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Return a CAR response that only matches the first variant's HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } ] @@ -991,31 +1666,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( mock_send_slack_error.assert_not_called() assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 - - # Verify the successfully matched variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" - - # Verify annotation statuses: 1 success, 3 failed - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", - ) - ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 + + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" + + # All 4 variant annotations succeeded + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", ) ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(present_events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1046,22 +1712,17 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno dummy_variant_mapping_job_run, ) - # Return a CAR response where: first variant succeeds, second has explicit CAR error, rest are silent failures - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) - second_hgvs = get_hgvs_from_post_mapped(mapped_variants[1].post_mapped) + # All 4 variants share 1 allele. CAR returns an explicit error for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + allele_hgvs = get_hgvs_from_post_mapped(alleles[0].post_mapped) registered_alleles_mock = [ - { - "@id": f"CA{mapped_variants[0].id}", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], - }, { "errorType": "InvalidHGVS", - "hgvs": second_hgvs, + "hgvs": allele_hgvs, "message": "The HGVS string is invalid.", "description": "error", - "inputLine": second_hgvs, + "inputLine": allele_hgvs, "position": "0", }, ] @@ -1079,31 +1740,19 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno standalone_worker_context, submit_score_set_mappings_to_car_sample_job_run.id ) - # Verify the variant whose HGVS returned an explicit CAR error has error details in annotation_metadata. - # Only 1 annotation should have EXTERNAL_SERVICE_REJECTED since only one CAR error was in the response. - car_rejected_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_service_rejected", - ) - ).all() - assert len(car_rejected_annotations) == 1 - rejected = car_rejected_annotations[0] - assert rejected.annotation_metadata["submitted_hgvs"] == second_hgvs - assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" - assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." - - # The remaining 2 failures (variants 3 and 4) got no CAR response — silent failures get EXTERNAL_API_ERROR. - silent_failure_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_api_error", + # The 1 shared allele was rejected by ClinGen → one failed event (reason service_rejected). + rejected_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.reason == "service_rejected", ) ).all() - assert len(silent_failure_annotations) == 2 - for ann in silent_failure_annotations: - assert ann.annotation_metadata["submitted_hgvs"] is not None - assert "car_error_type" not in ann.annotation_metadata + assert len(rejected_events) == 1 + for event in rejected_events: + assert event.disposition == "failed" + assert event.event_metadata["submitted_hgvs"] == allele_hgvs + assert event.event_metadata["car_error_type"] == "InvalidHGVS" + assert event.event_metadata["car_error_message"] == "The HGVS string is invalid." async def test_submit_score_set_mappings_to_car_propagates_exception_to_decorator( self, @@ -1185,15 +1834,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } - for mv in mapped_variants ] with ( @@ -1214,19 +1862,18 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( session.refresh(submit_score_set_mappings_to_car_sample_job_run) assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( self, @@ -1255,15 +1902,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": [get_hgvs_from_post_mapped(alleles[0].post_mapped)]}], } - for mv in mapped_variants ] with ( @@ -1288,19 +1934,18 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 per-variant annotations — all success + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_independent( self, @@ -1350,15 +1995,15 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.ERRORED assert submit_score_set_mappings_to_car_sample_job_run.error_message == "ClinGen service error" - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_pipeline( self, @@ -1413,15 +2058,15 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 @pytest.mark.unit @@ -1771,11 +2416,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -1842,11 +2487,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -1949,11 +2594,11 @@ async def dummy_no_linked_alleles_submission(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -1997,7 +2642,7 @@ async def test_submit_score_set_mappings_to_ldh_hgvs_not_found( # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2052,11 +2697,11 @@ async def dummy_submission_failure(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database # TODO:XXX: Change status to 'failed' once decorator supports it @@ -2122,15 +2767,15 @@ async def dummy_partial_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 success_count = 0 failure_count = 0 for ann in annotation_statuses: - if ann.status == "success": + if ann.disposition == "present": success_count += 1 - elif ann.status == "failed": + elif ann.disposition == "failed": failure_count += 1 assert success_count == 1 @@ -2200,11 +2845,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2277,11 +2922,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2349,11 +2994,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -2408,7 +3053,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2463,7 +3108,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index a55eb6b88..aa744e9c8 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -4,11 +4,14 @@ pytest.importorskip("arq") +from datetime import date from unittest.mock import AsyncMock, patch from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import JobStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.external_services.clingen_cache import warm_clingen_cache @@ -17,6 +20,55 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def _make_allele_with_caid( + session, score_set_id: int, variant_number: int, caid: str | None, vrs_digest: str +) -> Allele: + """Create a Variant → MappingRecord → Allele chain and return the Allele. + + get_alleles_for_score_set joins: Allele ← MappingRecordAllele (current) ← MappingRecord (current) ← Variant. + Leaving valid_to=NULL on MappingRecord and MappingRecordAllele makes them current. + """ + variant = Variant( + urn=f"{session.get(ScoreSet, score_set_id).urn}#{variant_number}", + score_set_id=score_set_id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.flush() + + allele = Allele( + vrs_digest=vrs_digest, + level="genomic", + post_mapped={"type": "Allele", "location": {"sequenceReference": {"refgetAccession": vrs_digest}}}, + clingen_allele_id=caid, + ) + session.add(allele) + session.flush() + + mapping_record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", + mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(mapping_record) + session.flush() + + link = MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=allele.id, + is_authoritative=True, + ) + session.add(link) + session.flush() + + return allele + + @pytest.mark.unit @pytest.mark.asyncio class TestWarmClingenCacheUnit: @@ -49,32 +101,28 @@ async def test_warms_cache_for_variants_with_caids( """Job calls get_clingen_allele_data for each distinct allele ID.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) - # Create two variants with the same CAID — should only warm once (distinct) - for i, caid in enumerate(["CA111111", "CA222222", "CA111111"]): - variant = Variant( - urn=f"urn:variant:warm-test-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 1}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 1}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + # Three variants, two sharing the same CAID — should only warm 2 distinct IDs. + # Two separate allele rows are needed (different VRS digests) to get 2 distinct CAIDs. + _make_allele_with_caid(session, score_set.id, 11, "CA111111", "digest-warm-0") + _make_allele_with_caid(session, score_set.id, 12, "CA222222", "digest-warm-1") + # Third variant points to same allele as first (same CAID CA111111 via a fresh allele row + # with the same digest — but since get_alleles_for_score_set returns per-variant rows and + # caid dedup happens in the warmer, we can share the digest). + _make_allele_with_caid(session, score_set.id, 13, "CA111111", "digest-warm-2") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) + mock_refresh_expiry = AsyncMock(return_value=True) - with patch( - "mavedb.worker.jobs.external_services.clingen_cache.get_clingen_allele_data", - mock_get_allele_data, + with ( + patch( + "mavedb.worker.jobs.external_services.clingen_cache.get_clingen_allele_data", + mock_get_allele_data, + ), + patch( + "mavedb.worker.jobs.external_services.clingen_cache.refresh_clingen_allele_data_expiry", + mock_refresh_expiry, + ), ): result = await warm_clingen_cache( mock_worker_ctx, @@ -83,6 +131,8 @@ async def test_warms_cache_for_variants_with_caids( ) assert result.status == JobStatus.SUCCEEDED + # A hit keeps its first writer's expiry, so every warmed entry's expiry is restarted. + assert {call.args[0] for call in mock_refresh_expiry.call_args_list} == {"CA111111", "CA222222"} # Should be called exactly 2 times (CA111111 and CA222222, deduplicated) assert mock_get_allele_data.call_count == 2 called_ids = {call.args[0] for call in mock_get_allele_data.call_args_list} @@ -100,24 +150,8 @@ async def test_skips_null_and_multi_variant_caids( caids = ["CA333333", None, "CA-MULTI-001,CA-MULTI-002"] for i, caid in enumerate(caids): - variant = Variant( - urn=f"urn:variant:warm-filter-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 10}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 10}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, 100 + i, caid, f"digest-filter-{i}") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -147,24 +181,8 @@ async def test_continues_on_individual_fetch_failure( score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) for i, caid in enumerate(["CA444444", "CA555555"]): - variant = Variant( - urn=f"urn:variant:warm-fail-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 20}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 20}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, 100 + i, caid, f"digest-fail-{i}") + session.commit() # First call raises, second succeeds mock_get_allele_data = AsyncMock( @@ -192,36 +210,80 @@ async def test_only_warms_current_mapped_variants( with_warm_clingen_cache_job, sample_warm_clingen_cache_job_run, ): - """Job only fetches allele IDs from current (not superseded) mapped variants.""" + """Job only fetches allele IDs from current (not superseded) mapping records.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) variant = Variant( - urn="urn:variant:warm-current-test", + urn=f"{score_set.urn}#11", score_set_id=score_set.id, hgvs_nt="NM_000000.1:c.30A>G", hgvs_pro="NP_000000.1:p.Met30Val", data={}, ) session.add(variant) - session.commit() + session.flush() - # Non-current mapped variant should be ignored - old_mv = MappedVariant( - variant_id=variant.id, + # Superseded allele (MappingRecord with valid_to set — not current) + old_allele = Allele( + vrs_digest="digest-old-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA666666", - current=False, - mapped_date="2023-01-01T00:00:00Z", - mapping_api_version="0.9.0", ) - # Current mapped variant should be included - current_mv = MappedVariant( + session.add(old_allele) + session.flush() + + from datetime import datetime, timezone + + old_mapping_record = MappingRecord( variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", + mapping_api_version="0.9.0", + mapped_date=date(2023, 1, 1), + ) + # Close this record so it is non-current (valid_to set) + old_mapping_record.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_mapping_record) + session.flush() + + old_link = MappingRecordAllele( + mapping_record_id=old_mapping_record.id, + score_set_id=old_mapping_record.score_set_id, + allele_id=old_allele.id, + is_authoritative=True, + ) + old_link.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_link) + session.flush() + + # Current allele (MappingRecord with valid_to=NULL — current) + current_allele = Allele( + vrs_digest="digest-current-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA777777", - current=True, - mapped_date="2024-01-01T00:00:00Z", + ) + session.add(current_allele) + session.flush() + + current_mapping_record = MappingRecord( + variant_id=variant.id, + score_set_id=variant.score_set_id, + assay_level="genomic", mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(current_mapping_record) + session.flush() + + current_link = MappingRecordAllele( + mapping_record_id=current_mapping_record.id, + score_set_id=current_mapping_record.score_set_id, + allele_id=current_allele.id, + is_authoritative=True, ) - session.add_all([old_mv, current_mv]) + session.add(current_link) session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 1ac616ffc..caa7e3f1f 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -3,19 +3,20 @@ import pytest import requests -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, FailureCategory, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - pytest.importorskip("arq") from unittest.mock import patch +from sqlalchemy import select + from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager @@ -26,6 +27,7 @@ "GeneSymbol": "TEST", "ClinicalSignificance": "benign", "ReviewStatus": "reviewed by expert panel", + "VariationID": "987654", }, } @@ -33,32 +35,29 @@ @pytest.mark.unit @pytest.mark.asyncio class TestRefreshClinvarControlsUnit: - """Tests for the refresh_clinvar_controls job function.""" + """Unit tests for the allele-model refresh_clinvar_controls job.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" + """Pin _generate_clinvar_versions to a single version so clinvar_version == '01_2026'.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_skips_version_on_fetch_failure( + async def test_no_alleles_with_caids( self, mock_worker_ctx, session, + with_populated_domain_data, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): - """Test that a fetch failure for a version is logged and skipped, not propagated.""" - - async def awaitable_exception(*args, **kwargs): - raise Exception("Network error") - + """No current alleles carry a CAID -> the job succeeds with nothing to do.""" with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=awaitable_exception, + return_value=MOCK_CLINVAR_DATA, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -68,20 +67,26 @@ async def awaitable_exception(*args, **kwargs): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarControl)).all() == [] + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] - async def test_refresh_clinvar_controls_no_mapped_variants( + async def test_fetch_failure_skips_version( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when there are no mapped variants.""" + """A version whose TSV fetch fails is logged and skipped, not propagated.""" + + async def boom(*args, **kwargs): + raise Exception("Network error") with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value={}, + side_effect=boom, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -89,35 +94,21 @@ async def test_refresh_clinvar_controls_no_mapped_variants( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - async def test_refresh_clinvar_controls_no_variants_have_caids( + async def test_multi_variant_caid_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.2G>A", - hgvs_pro="NP_000000.1:p.Val2Ile", - data={"hgvs_c": "NM_000000.1:c.2G>A", "hgvs_p": "NP_000000.1:p.Val2Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) + """An allele whose CAID is a multi-variant identifier is skipped (ClinVar can't key on it).""" + _, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" session.commit() with patch( @@ -130,34 +121,40 @@ async def test_refresh_clinvar_controls_no_variants_have_caids( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." + # Allele-keyed event: a cis-block CAID structurally can't key ClinVar — a pipeline gap, not a + # statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.MULTI_VARIANT_CAID + assert event.allele_id == allele.id and event.variant_id is None - async def test_refresh_clinvar_controls_variants_are_multivariants( + async def test_protein_level_allele_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when all variants are multi-variant CAIDs.""" - # Update the mapped variant to have a multi-variant CAID - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" + """A protein-level allele is never linked to ClinVar (a nucleotide-level DB): its clinical + calls come from its g./c. siblings. Skipped as not-applicable before any resolution runs.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" session.commit() - with patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + with ( + patch( + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value="VCV000000123", + ) as resolve_spy, + patch( + "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", + return_value=MOCK_CLINVAR_DATA, + ), ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -165,37 +162,35 @@ async def test_refresh_clinvar_controls_variants_are_multivariants( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) - - async def test_refresh_clinvar_controls_clingen_api_failure( + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(ClinvarControl)).all() == [] + # Short-circuited before touching ClinGen — a protein allele can never resolve to a ClinVar id. + resolve_spy.assert_not_awaited() + + # Allele-keyed not-applicable event: ClinVar is nucleotide-level, so a protein allele is a + # structural gap (like a multi-variant CAID), not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + + async def test_no_associated_clinvar_allele_id_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles ClinGen API failures gracefully.""" + """ClinGen returns no ClinVar allele id for the CAID -> absent/no_record event, no link.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to raise an exception with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), + return_value=None, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -208,40 +203,33 @@ async def test_refresh_clinvar_controls_clingen_api_failure( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - - # Verify an annotation status was created for the variant due to ClinGen API failure - mapped_variant = session.query(MappedVariant).first() - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + # ClinGen had no ClinVar AlleleID for this CAID — an informative negative about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None + + async def test_clinvar_data_not_found_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no associated ClinVar Allele ID gracefully.""" + """The resolved ClinVar allele id is absent from the version's TSV -> absent/no_record, no link.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return None with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + return_value="VCV000000123", ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value={"VCV000000999": {"GeneSymbol": "X", "ClinicalSignificance": "y", "ReviewStatus": "z"}}, ), ): result = await refresh_clinvar_controls( @@ -250,48 +238,34 @@ async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message - - async def test_refresh_clinvar_controls_no_clinvar_data_found( + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + # The CAID resolved to a ClinVar AlleleID, but it's absent from this release's snapshot — a + # genuine, version-scoped negative. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None + + async def test_clingen_api_failure_fails_when_total( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no ClinVar data found for the associated ClinVar Allele ID.""" - - # TSV data with a different allele ID than the one being looked up - non_matching_clinvar_data = { - "VCV000000001": { - "GeneSymbol": "TEST", - "ClinicalSignificance": "benign", - "ReviewStatus": "reviewed by expert panel", - }, - } + """A ClinGen API error with no successful links returns FAILED/DEPENDENCY_FAILURE.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=requests.exceptions.RequestException("ClinGen API error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=non_matching_clinvar_data, + return_value=MOCK_CLINVAR_DATA, ), ): result = await refresh_clinvar_controls( @@ -300,31 +274,24 @@ async def test_refresh_clinvar_controls_no_clinvar_data_found( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message + assert result.status == JobStatus.FAILED + assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.allele_id == allele.id and event.variant_id is None - async def test_refresh_clinvar_controls_successful_annotation_existing_control( + async def test_successful_link_new_control( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data.""" + """A resolved + present CAID creates a ClinvarControl, a live link, and a present/created event.""" + _, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", @@ -341,54 +308,53 @@ async def test_refresh_clinvar_controls_successful_annotation_existing_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - mapped_variant = session.query(MappedVariant).first() - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - async def test_refresh_clinvar_controls_successful_annotation_new_control( + assert result.data["created_link_count"] == 1 + + control = session.scalars(select(ClinvarControl)).one() + assert control.db_identifier == "VCV000000123" + assert control.db_version == "01_2026" + assert control.clinical_significance == "benign" + # AlleleID stays in db_identifier; the canonical VariationID is captured alongside it. + assert control.clinvar_variation_id == "987654" + + links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id, ClinvarAlleleLink.current) + ).all() + assert len(links) == 1 + assert links[0].clinvar_control_id == control.id + + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.PRESENT + assert event.reason == EventReason.CREATED + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.allele_id == allele.id and event.variant_id is None + + async def test_links_and_annotates_full_allele_set( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_rt_derived_allele_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant and mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.3C>T", - hgvs_pro="NP_000000.1:p.Ala3Val", - data={"hgvs_c": "NM_000000.1:c.3C>T", "hgvs_p": "NP_000000.1:p.Ala3Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA124", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + """ClinVar linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the matching CAID and is linked. Events are now allele-keyed, so the RT-derived + allele's present status is recorded directly — the limitation the per-variant bandaid had + (dropping the RT-derived allele's status) is lifted. Each allele gets its own event: present for + the linked RT allele, absent for the unmatched authoritative one. + """ + _, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + rt_caid = rt_allele.clingen_allele_id + + async def resolve(caid): + # Only the RT-derived allele's CAID resolves to a ClinVar id present in the TSV. + return "VCV000000123" if caid == rt_caid else None - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=resolve, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -401,1105 +367,327 @@ async def test_refresh_clinvar_controls_successful_annotation_new_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - async def test_refresh_clinvar_controls_idempotent_run( + # The RT-derived allele IS linked — the core fix. + rt_links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == rt_allele.id, ClinvarAlleleLink.current) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no ClinVar match, so it is not linked. + assert ( + session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == authoritative_allele.id) + ).all() + == [] + ) + + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, the + # unmatched authoritative allele is absent. The variant resolves its derived allele's status + # through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == AnnotationType.CLINVAR_CONTROL for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == Disposition.PRESENT + assert by_allele[rt_allele.id].reason == EventReason.CREATED + assert by_allele[authoritative_allele.id].disposition == Disposition.ABSENT + assert by_allele[authoritative_allele.id].reason == EventReason.NO_RECORD + + async def test_checkpoints_progress_within_a_version( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_rt_derived_allele_with_caid, ): - """Test that running the job multiple times does not create duplicate annotation statuses.""" + """A version with many uncached ClinGen lookups keeps the stall reaper's heartbeat fresh.""" + manager = JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id) - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( + patch("mavedb.worker.jobs.external_services.clinvar.CLINVAR_PROGRESS_EVERY_ALLELES", 1), patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + return_value=None, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], + return_value=MOCK_CLINVAR_DATA, ), + patch.object(manager, "update_progress", wraps=manager.update_progress) as update_progress, ): - # First run - result1 = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + result = await refresh_clinvar_controls( + mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, manager ) - session.commit() - - # Second run - result2 = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), - ) + assert result.status == JobStatus.SUCCEEDED + within_version = [call.args[2] for call in update_progress.call_args_list if "alleles." in call.args[2]] + assert within_version == [ + "Processing ClinVar version 01_2026 (1/1): 1/2 alleles.", + "Processing ClinVar version 01_2026 (1/1): 2/2 alleles.", + ] - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - async def test_refresh_clinvar_controls_partial_failure( + async def test_idempotent_rerun_skips_and_does_not_duplicate( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.4G>C", - hgvs_pro="NP_000000.1:p.Gly4Ala", - data={"hgvs_c": "NM_000000.1:c.4G>C", "hgvs_p": "NP_000000.1:p.Gly4Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA125", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA125": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" + """A second run finds the allele already linked at the version, skips the resolution, reports + preexisting, and creates neither a duplicate control nor a duplicate link.""" + _, allele = setup_sample_alleles_with_caid with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, - ), + return_value="VCV000000123", + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls( + first = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) + session.commit() + assert resolve_spy.await_count == 1 - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - async def test_total_api_failure_returns_failed( + second = await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + # Second run skipped the resolution entirely (version-keyed skip). + assert resolve_spy.await_count == 1 + + assert first.data["created_link_count"] == 1 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + + # One control, one live link — no churn. + assert len(session.scalars(select(ClinvarControl)).all()) == 1 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Per-version events: two allele-keyed events (one per run), no current flag. The first run + # created the link; the second found it preexisting. Latest event (by id) wins. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.allele_id == allele.id).order_by(AnnotationEvent.id) + ).all() + assert len(events) == 2 + assert events[0].reason == EventReason.CREATED + assert events[1].reason == EventReason.PREEXISTING + assert all(e.disposition == Disposition.PRESENT for e in events) + + async def test_release_reresolution_supersedes_newest_wins( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job returns FAILED when all ClinVar lookups fail.""" + """Defensive guard: if an allele re-resolves to a *different* control within the same release + (should never happen — archival data is immutable), the old live link is superseded + newest-wins, leaving exactly one live link per (allele, version).""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } + session.commit() + + tsv = { + "VCV000000123": { + "GeneSymbol": "A", + "ClinicalSignificance": "benign", + "ReviewStatus": "ok", + "VariationID": "111", + }, + "VCV000000999": { + "GeneSymbol": "B", + "ClinicalSignificance": "pathogenic", + "ReviewStatus": "ok", + "VariationID": "222", + }, + } + with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), + side_effect=["VCV000000123", "VCV000000999"], ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value=tsv, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + session.commit() + await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - - async def test_upsert_does_not_create_duplicate_control_when_row_already_exists( + control_a = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000123")).one() + control_b = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000999")).one() + + all_links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + live_links = [link for link in all_links if link.valid_to is None] + + # Exactly one live link per (allele, version) — the new control — and the old one is retired. + assert len(live_links) == 1 + assert live_links[0].clinvar_control_id == control_b.id + retired = [link for link in all_links if link.clinvar_control_id == control_a.id] + assert len(retired) == 1 + assert retired[0].valid_to is not None + # Gap-free handoff: the retired link's valid_to equals the successor's valid_from. + assert retired[0].valid_to == live_links[0].valid_from + # Stamped from the DB clock (func.now()), so the boundary is timezone-aware and comparable to + # every other func.now()-stamped valid-time row — a regression to a naive datetime.now() would + # land here as a tz-naive value. + assert live_links[0].valid_from.tzinfo is not None + assert retired[0].valid_to.tzinfo is not None + + async def test_force_reresolves_without_duplicating_link( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the upsert handles a pre-existing ClinicalControl row without creating a duplicate. - - This covers the concurrent-job race condition where two refresh_clinvar_controls - jobs run simultaneously for different score sets and both try to insert a - ClinicalControl for the same (db_name, db_identifier, db_version). The second - job's upsert must hit ON CONFLICT DO UPDATE rather than inserting a second row. - """ - # Simulate the state left by a concurrent job: the ClinicalControl row - # for this identifier/version already exists in the DB with stale data. - pre_existing_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - db_version="01_2026", - gene_symbol="OLD_SYMBOL", - clinical_significance="likely pathogenic", - clinical_review_status="criteria provided, single submitter", - ) - session.add(pre_existing_control) + """force bypasses the version-keyed skip and re-resolves, but the get-or-create link write + does not duplicate an existing live link.""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } session.commit() with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", return_value="VCV000000123", - ), + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + session.commit() + second = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.SUCCEEDED - - # Only one row should exist — the upsert must not have inserted a second one. - controls = session.query(ClinicalControl).filter_by(db_identifier="VCV000000123", db_version="01_2026").all() - assert len(controls) == 1 - - # The upsert should have updated the stale data from the concurrent job. - session.refresh(controls[0]) - assert controls[0].gene_symbol == "TEST" - assert controls[0].clinical_significance == "benign" - assert controls[0].clinical_review_status == "reviewed by expert panel" + # force re-resolved on the second run despite an existing link. + assert resolve_spy.await_count == 2 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None @pytest.mark.integration @pytest.mark.asyncio -class TestRefreshClinvarControlsIntegration: - """Integration tests for the refresh_clinvar_controls job function.""" +class TestRefreshClinvarControlsArqContext: + """End-to-end tests for refresh_clinvar_controls within an ARQ worker context.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when there are no mapped variants.""" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify no controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_variants_with_caid( + async def test_arq_context_successful_link( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.5T>A", - hgvs_pro="NP_000000.1:p.Leu5Gln", - data={"hgvs_c": "NM_000000.1:c.5T>A", "hgvs_p": "NP_000000.1:p.Leu5Gln"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - + """The job links an allele and records a present event under an ARQ worker.""" with ( patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value="VCV000000123", ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controlsvariants_are_multivariants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when all variants are multi-variant CAIDs.""" - # Add a variant with a multi-variant CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-multicid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.6A>G", - hgvs_pro="NP_000000.1:p.Thr6Ala", - data={"hgvs_c": "NM_000000.1:c.6A>G", "hgvs_p": "NP_000000.1:p.Thr6Ala"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA-MULTI-003,CA-MULTI-004", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - with ( patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) + assert len(session.scalars(select(ClinvarControl)).all()) >= 1 + assert len(session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.current)).all()) == 1 - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == Disposition.PRESENT + assert events[0].annotation_type == AnnotationType.CLINVAR_CONTROL + assert events[0].allele_id is not None and events[0].variant_id is None - # Verify job run status is marked as completed session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + async def test_arq_context_exception_handling( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job handles no associated ClinVar Allele ID gracefully.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.7C>A", - hgvs_pro="NP_000000.1:p.Ser7Tyr", - data={"hgvs_c": "NM_000000.1:c.7C>A", "hgvs_p": "NP_000000.1:p.Ser7Tyr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA126", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return None + """An unexpected error during resolution is caught by the decorators and the job errors.""" with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + side_effect=ValueError("Unexpected error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_slack, ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + mock_slack.assert_called_once() + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_clinvar_data( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job handles no ClinVar data found for the associated ClinVar Allele ID.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.8G>T", - hgvs_pro="NP_000000.1:p.Val8Phe", - data={"hgvs_c": "NM_000000.1:c.8G>T", "hgvs_p": "NP_000000.1:p.Val8Phe"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA127", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000001", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_existing_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.9A>C", - hgvs_pro="NP_000000.1:p.Lys9Thr", - data={"hgvs_c": "NM_000000.1:c.9A>C", "hgvs_p": "NP_000000.1:p.Lys9Thr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA128", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - clinical_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - clinical_significance="likely pathogenic", - gene_symbol="TEST", - clinical_review_status="criteria provided, single submitter", - db_version="01_2026", - ) - session.add(clinical_control) - session.commit() - - mapped_variant.clinical_controls.append(clinical_control) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was updated - session.refresh(clinical_control) - assert clinical_control.clinical_significance == "benign" - assert clinical_control.clinical_review_status == "reviewed by expert panel" - assert mapped_variant in clinical_control.mapped_variants - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_new_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.10C>G", - hgvs_pro="NP_000000.1:p.Pro10Arg", - data={"hgvs_c": "NM_000000.1:c.10C>G", "hgvs_p": "NP_000000.1:p.Pro10Arg"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA129", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_pipeline_context( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_pipeline, - sample_refresh_clinvar_controls_job_in_pipeline, - ): - """Integration test: job successfully annotates a variant with ClinVar control data in a pipeline context.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_in_pipeline.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.12G>A", - hgvs_pro="NP_000000.1:p.Met12Ile", - data={"hgvs_c": "NM_000000.1:c.12G>A", "hgvs_p": "NP_000000.1:p.Met12Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_in_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_in_pipeline) - assert sample_refresh_clinvar_controls_job_in_pipeline.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - session.refresh(sample_refresh_clinvar_controls_pipeline) - assert sample_refresh_clinvar_controls_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_idempotent_run( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: running the job multiple times does not create duplicate annotation statuses.""" - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], - ), - ): - # First run - result1 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - session.commit() - # reset the job run status to pending for the second run - sample_refresh_clinvar_controls_job_run.status = JobStatus.PENDING - session.commit() - - # Second run - result2 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_partial_failure( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:integration-test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.11G>C", - hgvs_pro="NP_000000.1:p.Gly11Ala", - data={"hgvs_c": "NM_000000.1:c.11G>C", "hgvs_p": "NP_000000.1:p.Gly11Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA130": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - # Verify a clinical control was added for the successfully annotated variant and not the unsuccessful one - clinical_control1 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant1)).one() - ) - assert clinical_control1.db_identifier == "VCV000000123" - - clinical_control2 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant2)).all() - ) - assert len(clinical_control2) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_propagates_exceptions_to_decorator( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Test that unexpected exceptions are propagated.""" - - # Mock the get_associated_clinvar_allele_id function to raise an unexpected exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestRefreshClinvarControlsArqContext: - """Tests for running the refresh_clinvar_controls job function within an ARQ worker context.""" - - @pytest.fixture(autouse=True) - def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" - with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", - return_value=[(2026, 1)], - ): - yield - - async def test_refresh_clinvar_controls_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context in a pipeline context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - pass - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context in a pipeline context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - # Verify the pipeline is marked as failed - pass diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index fc8e211c0..03a3cefac 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -4,13 +4,20 @@ pytest.importorskip("arq") -from unittest.mock import MagicMock, patch +from unittest.mock import patch +from sqlalchemy import select + +from mavedb.lib import gnomad as gnomad_lib +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.gnomad import link_gnomad_variants from mavedb.worker.lib.managers.job_manager import JobManager @@ -22,7 +29,7 @@ class TestLinkGnomadVariantsUnit: """Unit tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -30,7 +37,7 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test linking gnomAD variants when no mapped variants have CAIDs.""" + """No authoritative alleles with CAIDs -> the job succeeds with nothing to do.""" result = await link_gnomad_variants( mock_worker_ctx, 1, @@ -47,7 +54,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test linking gnomAD variants when no gnomAD variants match the CAIDs.""" @@ -55,7 +62,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value={}, + return_value=[], ), patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -68,6 +75,44 @@ async def test_link_gnomad_variants_no_gnomad_matches( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_protein_level_allele_skipped( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """A protein-level allele is never linked to gnomAD (a nucleotide-level, genomic-coordinate + resource): its frequencies come from its g./c. siblings. Skipped as not-applicable, and it + never reaches the Athena query — sparing the round-trip.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" + session.commit() + + with patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + ) as fetch_spy: + result = await link_gnomad_variants( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_link_gnomad_variants_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(GnomadAlleleLink)).all() == [] + # The protein allele's CAID was dropped from the query set, so Athena is never queried. + fetch_spy.assert_not_called() + + # Allele-keyed not-applicable event: gnomAD is nucleotide-level, so a protein allele is a + # structural gap, not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.GNOMAD_ALLELE_FREQUENCY + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + async def test_link_gnomad_variants_call_linking_method( self, session, @@ -75,7 +120,7 @@ async def test_link_gnomad_variants_call_linking_method( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that the linking method is called when gnomAD variants match CAIDs.""" @@ -83,11 +128,11 @@ async def test_link_gnomad_variants_call_linking_method( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value=[MagicMock()], + return_value=[object()], ), patch( - "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_mapped_variants", - return_value=1, + "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_alleles", + return_value={}, ) as mock_linking_method, patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -108,7 +153,7 @@ async def test_link_gnomad_variants_propagates_exceptions( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are propagated.""" @@ -134,7 +179,7 @@ async def test_link_gnomad_variants_propagates_exceptions( class TestLinkGnomadVariantsIntegration: """Integration tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -142,19 +187,18 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test the end-to-end functionality of the link_gnomad_variants job when no variants have CAIDs.""" + """Test the end-to-end functionality of the link_gnomad_variants job when no alleles have CAIDs.""" result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify no annotations were rendered (since there were no variants with CAIDs) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + # Verify no annotations were rendered (since there were no alleles with CAIDs) + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -167,13 +211,13 @@ async def test_link_gnomad_variants_no_matching_caids( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job when no matching CAIDs are found.""" - # Update the created mapped variant to have a CAID that won't match any gnomAD data - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "NON_MATCHING_CAID" + # Update the allele to have a CAID that won't match any seeded gnomAD data + _, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "NON_MATCHING_CAID" session.commit() # Patch the athena engine to use the mock athena_engine fixture @@ -183,15 +227,16 @@ async def test_link_gnomad_variants_no_matching_caids( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify a skipped annotation status was rendered (since there were variants with CAIDs) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + # Verify an absent event was rendered: the allele's CAID was queried but gnomAD had no record. + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id == allele.id and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -204,10 +249,11 @@ async def test_link_gnomad_variants_successful_linking_independent( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking.""" + _, allele = setup_sample_alleles_with_caid # Patch the athena engine to use the mock athena_engine fixture with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): @@ -216,20 +262,173 @@ async def test_link_gnomad_variants_successful_linking_independent( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that a gnomAD variant was created and a live link to the allele established + assert len(session.scalars(select(GnomADVariant)).all()) > 0 + live_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(live_links) == 1 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) assert sample_link_gnomad_variants_run.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_links_and_annotates_rt_derived_allele( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_rt_derived_allele_with_caid, + athena_engine, + ): + """gnomAD linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the CAID gnomAD matches and is linked. Events are now allele-keyed, so the + RT-derived allele's PRESENT status is recorded directly — the limitation the per-variant + bandaid had (dropping the RT-derived allele's status) is lifted. Each allele in the score set + gets its own event: present for the linked RT allele, absent for the unmatched authoritative + one.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + assert result.status == JobStatus.SUCCEEDED + + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == rt_allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no gnomAD match, so it gets no link. + assert ( + len( + session.scalars( + select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == authoritative_allele.id) + ).all() + ) + == 0 + ) + + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, + # the unmatched authoritative allele is absent. The variant resolves its derived allele's + # status through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "gnomad_allele_frequency" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" + + async def test_link_gnomad_variants_skips_allele_already_current( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """An allele already linked at the current gnomAD version is skipped: no Athena query, the + status reports SUCCESS/preexisting, and the existing link is not churned.""" + _, allele = setup_sample_alleles_with_caid + + # Simulate a prior run: a live link at the current gnomAD version. + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + with patch("mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids") as fetch_spy: + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + + # Link not churned: still exactly one, still live. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Event is PRESENT, marked preexisting (the link was already current; not created this run). + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" + + async def test_link_gnomad_variants_force_refetches_without_churn( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + athena_engine, + ): + """force bypasses the skip and re-fetches, but the linker supersedes only on change, so a + forced re-run of unchanged data reports preexisting and does not churn the link.""" + _, allele = setup_sample_alleles_with_caid + + # Prior live link pointing at the same variant the Athena mock resolves to (1-12345-G-A). + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=23, + allele_number=32432423, + allele_frequency=23 / 32432423, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + sample_link_gnomad_variants_run.job_params = {**sample_link_gnomad_variants_run.job_params, "force": True} + session.commit() + + with ( + patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), + patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + side_effect=gnomad_lib.gnomad_variant_data_for_caids, + ) as fetch_spy, + ): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_called_once() # force bypassed the version-keyed skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + # Unchanged → no churn. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + async def test_link_gnomad_variants_successful_linking_pipeline( self, session, @@ -237,7 +436,7 @@ async def test_link_gnomad_variants_successful_linking_pipeline( mock_worker_ctx, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking in a pipeline.""" @@ -249,15 +448,16 @@ async def test_link_gnomad_variants_successful_linking_pipeline( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -274,7 +474,7 @@ async def test_link_gnomad_variants_exceptions_handled_by_decorators( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are handled by decorators.""" @@ -317,7 +517,7 @@ async def test_link_gnomad_variants_with_arq_context_independent( with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture.""" @@ -328,15 +528,16 @@ async def test_link_gnomad_variants_with_arq_context_independent( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run) @@ -351,7 +552,7 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( athena_engine, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture in a pipeline.""" @@ -362,15 +563,16 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -389,7 +591,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -406,13 +608,12 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run) @@ -427,7 +628,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline athena_engine, sample_link_gnomad_variants_pipeline, sample_link_gnomad_variants_run_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -444,13 +645,12 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run_pipeline) diff --git a/tests/worker/jobs/external_services/test_hgvs.py b/tests/worker/jobs/external_services/test_hgvs.py deleted file mode 100644 index 946724cc5..000000000 --- a/tests/worker/jobs/external_services/test_hgvs.py +++ /dev/null @@ -1,544 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.hgvs import populate_hgvs_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - -SAMPLE_CA_ALLELE_DATA = { - "genomicAlleles": [ - { - "referenceGenome": "GRCh38", - "hgvs": ["NC_000001.11:g.12345A>G"], - } - ], - "transcriptAlleles": [ - { - "hgvs": ["NM_000000.1:c.1A>G"], - "proteinEffect": {"hgvs": "NP_000000.1:p.Met1Val"}, - "MANE": { - "nucleotide": {"RefSeq": {"hgvs": "NM_000000.1:c.1A>G"}}, - "protein": {"RefSeq": {"hgvs": "NP_000000.1:p.Met1Val"}}, - }, - } - ], -} - -SAMPLE_PA_ALLELE_DATA = { - "aminoAcidAlleles": [ - { - "hgvs": ["NP_000000.1:p.Met1Val"], - } - ], -} - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateHgvsForScoreSetUnit: - """Unit tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test populating HGVS when no mapped variants exist.""" - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - async def test_variant_without_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant without a CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_variant_with_multi_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant with a multi-variant CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = "CA123,CA456" - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_successful_ca_allele_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test successful HGVS population for a CA allele.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["populated_count"] == 1 - - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - session.refresh(mapped_variant) - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - async def test_clingen_api_error_recorded_as_failed( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that ClinGen API errors are recorded as failed annotations.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - - async def test_total_api_failure_sends_slack_alert( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a Slack alert is sent when all variants fail HGVS population.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - patch("mavedb.worker.jobs.external_services.hgvs.log_and_send_slack_message") as mock_slack, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - assert result.data["populated_count"] == 0 - mock_slack.assert_called_once() - - async def test_clingen_allele_not_found_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a 404 from ClinGen results in a skipped annotation.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=None, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetIntegration: - """Integration tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end successful HGVS population.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated with HGVS - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population_pipeline( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end HGVS population in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that variants without CAIDs get a skipped annotation status.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - sample_populate_hgvs_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetArqContext: - """Tests for populate_hgvs_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job completed - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_pipeline, - sample_populate_hgvs_run_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.ERRORED - - # Verify pipeline failed - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/external_services/test_uniprot.py b/tests/worker/jobs/external_services/test_uniprot.py index 8f1bf1304..8d58e607c 100644 --- a/tests/worker/jobs/external_services/test_uniprot.py +++ b/tests/worker/jobs/external_services/test_uniprot.py @@ -19,6 +19,8 @@ TEST_UNIPROT_ID_MAPPING_SWISS_PROT_RESPONSE, TEST_UNIPROT_SWISS_PROT_TYPE, VALID_NT_ACCESSION, + VALID_NT_ACCESSION_UNVERSIONED, + VALID_PRO_ACCESSION, VALID_UNIPROT_ACCESSION, ) @@ -162,7 +164,7 @@ async def test_submit_uniprot_mapping_jobs_no_jobs_submitted( # Verify that the job metadata contains no submitted jobs session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": None, "accession": VALID_NT_ACCESSION} + "1": {"job_id": None, "accession": VALID_NT_ACCESSION_UNVERSIONED} } async def test_submit_uniprot_mapping_jobs_api_failure_raises( @@ -239,7 +241,7 @@ async def test_submit_uniprot_mapping_jobs_raises_dependent_job_not_available( # Verify that the job metadata contains the submitted jobs (which were submitted before the error) session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION} + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED} } async def test_submit_uniprot_mapping_jobs_successful_submission( @@ -277,7 +279,7 @@ async def test_submit_uniprot_mapping_jobs_successful_submission( assert isinstance(job_result, JobExecutionOutcome) assert job_result.status == JobStatus.SUCCEEDED - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run) @@ -287,6 +289,59 @@ async def test_submit_uniprot_mapping_jobs_successful_submission( session.refresh(sample_dummy_polling_job_for_submission_run) assert sample_dummy_polling_job_for_submission_run.job_params["mapping_jobs"] == expected_submitted_jobs + @pytest.mark.parametrize( + "post_mapped_metadata, expected_from_db, expected_accession", + [ + ( + {"cdna": {"sequence_accessions": [VALID_NT_ACCESSION]}}, + "RefSeq_Nucleotide", + VALID_NT_ACCESSION_UNVERSIONED, + ), + ({"protein": {"sequence_accessions": [VALID_PRO_ACCESSION]}}, "RefSeq_Protein", VALID_PRO_ACCESSION), + ], + ) + async def test_submit_uniprot_mapping_jobs_submits_query_form_of_accession( + self, + session, + mock_worker_ctx, + with_populated_domain_data, + with_submit_uniprot_mapping_job, + with_dummy_polling_job_for_submission_run, + sample_score_set, + sample_submit_uniprot_mapping_jobs_run, + sample_dummy_polling_job_for_submission_run, + post_mapped_metadata, + expected_from_db, + expected_accession, + ): + """NM_ accessions are submitted unversioned, other accessions as-is, and the submitted + form is what the polling job receives to match results against.""" + target_gene = sample_score_set.target_genes[0] + target_gene.post_mapped_metadata = post_mapped_metadata + session.commit() + + with patch( + "mavedb.worker.jobs.external_services.uniprot.UniProtIDMappingAPI.submit_id_mapping", + return_value="job_12345", + ) as mock_submit: + job_result = await submit_uniprot_mapping_jobs_for_score_set( + mock_worker_ctx, + 1, + JobManager( + db=session, + redis=mock_worker_ctx["redis"], + job_id=sample_submit_uniprot_mapping_jobs_run.id, + ), + ) + + assert job_result.status == JobStatus.SUCCEEDED + mock_submit.assert_called_once_with(expected_from_db, "UniProtKB", [expected_accession]) + + session.refresh(sample_dummy_polling_job_for_submission_run) + assert sample_dummy_polling_job_for_submission_run.job_params["mapping_jobs"] == { + "1": {"job_id": "job_12345", "accession": expected_accession} + } + async def test_submit_uniprot_mapping_jobs_partial_submission( self, session, @@ -335,7 +390,7 @@ async def test_submit_uniprot_mapping_jobs_partial_submission( assert job_result.status == JobStatus.SUCCEEDED expected_submitted_jobs = { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}, + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}, "2": {"job_id": None, "accession": "NM_000546"}, } @@ -385,7 +440,7 @@ async def test_submit_uniprot_mapping_jobs_success_independent_ctx( assert isinstance(job_result, JobExecutionOutcome) assert job_result.status == JobStatus.SUCCEEDED - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run) @@ -436,7 +491,7 @@ async def test_submit_uniprot_mapping_jobs_success_pipeline_ctx( assert isinstance(job_result, JobExecutionOutcome) assert job_result.status == JobStatus.SUCCEEDED - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run_in_pipeline) @@ -662,7 +717,7 @@ async def test_submit_uniprot_mapping_jobs_no_jobs_submitted( # Verify that the job metadata contains no submitted jobs session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": None, "accession": VALID_NT_ACCESSION} + "1": {"job_id": None, "accession": VALID_NT_ACCESSION_UNVERSIONED} } # Verify that the submission job was completed successfully @@ -697,8 +752,8 @@ async def test_submit_uniprot_mapping_jobs_partial_submission( session.commit() # Add accessions to both target genes' post mapped metadata - for idx, tg in enumerate(sample_score_set.target_genes): - tg.post_mapped_metadata = {"protein": {"sequence_accessions": [VALID_NT_ACCESSION + f"{idx:05d}"]}} + for tg, accession in zip(sample_score_set.target_genes, [VALID_NT_ACCESSION, "NM_000546.6"]): + tg.post_mapped_metadata = {"protein": {"sequence_accessions": [accession]}} session.commit() with ( @@ -715,8 +770,8 @@ async def test_submit_uniprot_mapping_jobs_partial_submission( assert job_result.status == JobStatus.SUCCEEDED expected_submitted_jobs = { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION + "00000"}, - "2": {"job_id": None, "accession": VALID_NT_ACCESSION + "00001"}, + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}, + "2": {"job_id": None, "accession": "NM_000546"}, } # Verify that the job metadata contains both submitted and failed jobs @@ -765,7 +820,7 @@ async def test_submit_uniprot_mapping_jobs_no_dependent_job_raises( # Verify that the job metadata contains the job we submitted before the error session.refresh(sample_submit_uniprot_mapping_jobs_run) assert sample_submit_uniprot_mapping_jobs_run.metadata_["submitted_jobs"] == { - "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION} + "1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED} } # Verify that the submission job failed @@ -810,7 +865,7 @@ async def test_submit_uniprot_mapping_jobs_with_arq_context_independent( await arq_worker.async_run() await arq_worker.run_check() - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run) @@ -859,7 +914,7 @@ async def test_submit_uniprot_mapping_jobs_with_arq_context_pipeline( await arq_worker.async_run() await arq_worker.run_check() - expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION}} + expected_submitted_jobs = {"1": {"job_id": "job_12345", "accession": VALID_NT_ACCESSION_UNVERSIONED}} # Verify that the job metadata contains the submitted job session.refresh(sample_submit_uniprot_mapping_jobs_run_in_pipeline) diff --git a/tests/worker/jobs/external_services/test_variant_translation.py b/tests/worker/jobs/external_services/test_variant_translation.py deleted file mode 100644 index 3e1f364ea..000000000 --- a/tests/worker/jobs/external_services/test_variant_translation.py +++ /dev/null @@ -1,770 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from sqlalchemy import select - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.models.variant_translation import VariantTranslation -from mavedb.worker.jobs.external_services.variant_translation import populate_variant_translations_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -# --- Unit Tests --- - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateVariantTranslationsUnit: - """Unit tests for the populate_variant_translations_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test that the job succeeds with zero translations when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_variant_without_caid_no_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a variant without a CAID results in no translations.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_ca_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele creates translations via PA lookup.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00001"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA11111", "CA22222"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # 1 for PA00001->CA9765210 (the original CA), 2 for PA00001->CA11111 and PA00001->CA22222 - assert result.data["translations_created"] == 3 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 3 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation is not None - - async def test_pa_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele creates translations via CA lookup.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA99999" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA33333", "CA44444"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 2 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 - aa_ids = {t.aa_clingen_id for t in translations} - assert aa_ids == {"PA99999"} - - async def test_multi_variant_caid_expanded( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that comma-separated CAIDs are expanded and each processed independently.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "CA55555,CA66666" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00002"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # PA00002->CA55555 and PA00002->CA66666 - assert result.data["translations_created"] == 2 - - async def test_ca_allele_no_pa_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele with no canonical PA IDs results in a skip.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "skipped" - - async def test_pa_allele_no_ca_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele with no registered CA IDs results in a skip.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA88888" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - async def test_ca_allele_api_failure_records_failed_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a ClinGen API failure for CA allele records a failed annotation.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "failed" - - async def test_unrecognized_allele_format_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that an unrecognized allele ID format is skipped.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - - async def test_duplicate_translations_not_created( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that duplicate translations are not created on re-run.""" - # Pre-populate a translation - session.add(VariantTranslation(aa_clingen_id="PA00003", nt_clingen_id="CA9765210")) - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00003"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - translations = session.scalars( - select(VariantTranslation).where(VariantTranslation.aa_clingen_id == "PA00003") - ).all() - assert len(translations) == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - async def test_multiple_alleles_sharing_pa_no_duplicate_error( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that two CA alleles mapping to the same PA don't cause a UniqueViolation. - - This is a regression test for a bug where the SELECT-then-INSERT upsert pattern - failed to detect in-session duplicates: both alleles' iterations called - upsert_variant_translations with overlapping (PA, CA) pairs, the SELECT found - no committed row, both staged db.add() for the same pair, and the subsequent - update_progress commit raised a UniqueViolation. - """ - # Add a second variant with a different CA allele under the same score set. - score_set_id = sample_populate_variant_translations_run.job_params["score_set_id"] - - variant2 = Variant( - urn="urn:variant:test-second-ca-allele", - score_set_id=score_set_id, - hgvs_nt="NM_000000.1:c.2T>G", - hgvs_pro="NP_000000.1:p.Val2Gly", - data={}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA_SECOND", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Both CA alleles resolve to the same PA. The PA then returns the same set of - # registered CAs for both iterations, producing fully overlapping translation pairs. - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA_SHARED"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA9765210", "CA_SECOND"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - pairs = {(t.aa_clingen_id, t.nt_clingen_id) for t in translations} - # PA_SHARED paired with each CA: CA9765210 (original from allele 1), - # CA_SECOND (original from allele 2), plus both as registered CAs. - assert ("PA_SHARED", "CA9765210") in pairs - assert ("PA_SHARED", "CA_SECOND") in pairs - - async def test_total_api_failure_returns_failed( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job returns FAILED when all variant translation lookups fail.""" - import requests - - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - assert result.data["translations_created"] == 0 - - -# --- Integration Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsIntegration: - """Integration tests that exercise the full decorator stack.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_successful_job_updates_status( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a successful job run updates the job status to SUCCEEDED.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00004"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA77777"], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00004->CA9765210 and PA00004->CA77777 - - async def test_job_with_pipeline_updates_pipeline_status( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a job in a pipeline updates the pipeline status on success.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00005"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline.id, - ) - - session.refresh(sample_populate_variant_translations_run_pipeline) - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that variants without CAIDs produce no annotations (filtered before processing).""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_unrecognized_allele_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unrecognized allele formats create skipped annotations through the full stack.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - -# --- ARQ Context Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsArqContext: - """Tests for populate_variant_translations_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00006"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA88888"], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00006->CA9765210 and PA00006->CA88888 - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00007"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_pipeline, - sample_populate_variant_translations_run_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.ERRORED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 611c33894..cc24e213c 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -4,29 +4,76 @@ pytest.importorskip("arq") +from datetime import date, timedelta from unittest.mock import patch from sqlalchemy import select +from variant_annotation.lib.vep import ( + NO_CHANGE_TERM, + RESOLVER_VERSION, + ConsequenceOutcome, + ConsequenceResolution, + ConsequenceSource, + VepInput, +) + from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, JobStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.enums.vep import VepConsequenceSource +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.external_services.vep import populate_vep_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +# resolve_consequences lives in mavedb.lib.vep; the worker imports it, so we patch the worker's binding. +# The orchestration itself is unit-tested in tests/lib/test_vep.py::TestResolveConsequences. +_RESOLVE = "mavedb.worker.jobs.external_services.vep.resolve_consequences" +_RELEASE = "mavedb.worker.jobs.external_services.vep.get_ensembl_release" +_ENSEMBL_RELEASE = "116" + + +def _resolved(hgvs: str, term: str) -> ConsequenceResolution: + """A RESOLVED resolution keyed by hgvs — the shape resolve_consequences returns per input.""" + return ConsequenceResolution( + input=VepInput(hgvs=hgvs), + outcome=ConsequenceOutcome.RESOLVED, + consequence_terms=[term], + most_severe_consequence=term, + source=ConsequenceSource.TRANSCRIPT, + ) + + +def _errored(hgvs: str) -> ConsequenceResolution: + """An ERRORED resolution — the request failed, result unknown (must be retried, never linked).""" + return ConsequenceResolution(input=VepInput(hgvs=hgvs), outcome=ConsequenceOutcome.ERRORED, error="boom") + + +@pytest.fixture(autouse=True) +def mock_ensembl_release(): + """Stamp every job run with a fixed Ensembl release so tests version-key deterministically without + hitting /info/software. Tests exercising a release-fetch failure override this with an inner patch.""" + with patch(_RELEASE, return_value=_ENSEMBL_RELEASE): + yield + + +def _live_consequences_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() + @pytest.mark.asyncio @pytest.mark.unit class TestPopulateVepForScoreSetUnit: - """Unit tests for populate_vep_for_score_set.""" + """Unit tests for the populate_vep_for_score_set job.""" - async def test_no_mapped_variants( + async def test_no_alleles_with_hgvs( self, session, with_populated_domain_data, @@ -34,7 +81,7 @@ async def test_no_mapped_variants( mock_worker_ctx, sample_populate_vep_run, ): - """Job succeeds with zero counts when no mapped variants exist.""" + """No alleles with HGVS -> the job succeeds with nothing to do.""" result = await populate_vep_for_score_set( mock_worker_ctx, 1, @@ -43,405 +90,594 @@ async def test_no_mapped_variants( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 - assert result.data["variants_with_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 0 + assert result.data["created_allele_count"] == 0 + assert result.data["preexisting_allele_count"] == 0 + assert result.data["absent_allele_count"] == 0 + assert result.data["errored_allele_count"] == 0 - async def test_variant_without_hgvs_assay_level_skipped( + async def test_propagates_exceptions( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """A mapped variant with no hgvs_assay_level gets a SKIPPED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - mapped_variant.hgvs_assay_level = None - session.commit() + """Exceptions raised while resolving consequences are propagated.""" + with patch(_RESOLVE, side_effect=Exception("Test exception")): + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) + + assert str(exc_info.value) == "Test exception" + + async def test_aborts_when_release_unavailable( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """If the Ensembl release cannot be fetched the job aborts rather than mis-versioning its writes + — the version is load-bearing for the skip, so a failure must propagate, not be swallowed.""" + with ( + patch(_RELEASE, side_effect=Exception("info/software unavailable")), + patch(_RESOLVE) as mock_resolve, + ): + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + assert str(exc_info.value) == "info/software unavailable" + mock_resolve.assert_not_called() # never queried VEP without a version to stamp + +@pytest.mark.asyncio +@pytest.mark.integration +class TestPopulateVepForScoreSetIntegration: + """Integration tests for the populate_vep_for_score_set job.""" + + async def test_no_alleles_with_hgvs( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + ): + """End-to-end: no alleles -> no consequence rows, no annotations, job succeeds.""" + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SKIPPED - assert annotation.failure_category == AnnotationFailureCategory.MISSING_IDENTIFIER + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 - async def test_vep_api_success_sets_consequence_and_annotation( + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED + + async def test_no_consequence_resolved( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """VEP returns a consequence: mapped variant and SUCCESS annotation are updated.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level + """A genuine empty (VEP queried successfully, found nothing) writes no consequence row and an + ABSENT/no_record event — a trustworthy negative, distinct from a failed request.""" + _, allele = setup_sample_alleles_for_vep - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value={}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 1 - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 - - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert result.data["absent_allele_count"] == 1 + assert len(_live_consequences_for(session, allele.id)) == 0 + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None - async def test_vep_missing_triggers_variant_recoder_fallback( + async def test_skips_protein_level_allele( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_protein_allele_for_vep, ): - """When VEP misses a variant, Variant Recoder is called and its result fed back to VEP.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + """A protein-level allele is never submitted to VEP (#772 routing) — its consequence is carried + by the coding alleles reverse translation enumerates. With only a protein allele present the job + has nothing eligible to do: no external query, and no consequence row or event for it.""" + _, allele = setup_sample_protein_allele_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # initial VEP pass returns nothing - {genomic_hgvs: "missense_variant"}, # second VEP pass on recoded HGVS - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 + mock_resolve.assert_not_called() # protein allele routed out before any query + assert len(_live_consequences_for(session, allele.id)) == 0 + assert session.scalars(select(AnnotationEvent).where(AnnotationEvent.allele_id == allele.id)).all() == [] - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" - - async def test_variant_recoder_failure_annotated_as_failed( + async def test_request_failure_recorded_as_errored( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Variant Recoder returning no result for an HGVS produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep + """An allele whose VEP/Recoder request *failed* is FAILED/api_error — distinct from a genuine + empty — so a transient outage is never mistaken for a confirmed absence.""" + _, allele = setup_sample_alleles_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={}, # recoder has no result - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + # Resolver reports the allele's HGVS as errored (request failed), not as a hit or an empty. + with patch(_RESOLVE, return_value={allele.hgvs_c: _errored(allele.hgvs_c)}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 1 + assert result.data["errored_allele_count"] == 1 + assert result.data["absent_allele_count"] == 0 + assert len(_live_consequences_for(session, allele.id)) == 0 + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "failed" + assert events[0].reason == "api_error" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None + + async def test_successful_linking_independent( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """A resolved consequence creates a single live VepAlleleConsequence and a SUCCESS VAS row.""" + _, allele = setup_sample_alleles_for_vep - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - async def test_vep_failure_after_recoder_annotated_as_failed( + assert result.status == JobStatus.SUCCEEDED + assert result.data["created_allele_count"] == 1 + + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == _ENSEMBL_RELEASE + assert live[0].access_date == date.today() + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None + + async def test_links_and_annotates_rt_derived_allele( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_rt_derived_allele_for_vep, ): - """VEP returning no consequence even after Variant Recoder produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + """VEP linkage covers the full allele set, not just authoritative links: the RT-derived allele's + genomic HGVS resolves and gets a consequence row. Events are allele-keyed, so each allele in the + score set gets its own event — present for the resolved RT allele, absent/no_record for the + authoritative allele VEP queried but found nothing (a genuine empty, distinct from a request + failure). The per-variant bandaid's limitation (dropping the RT-derived allele's status) is lifted.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep + + # VEP resolves only the RT-derived allele's genomic HGVS; the authoritative allele's coding + # HGVS is queried successfully but yields no consequence this run (a genuine empty, not errored). + with patch(_RESOLVE, return_value={rt_allele.hgvs_g: _resolved(rt_allele.hgvs_g, "missense_variant")}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, # VEP returns nothing in both passes - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + assert result.status == JobStatus.SUCCEEDED + + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_live = _live_consequences_for(session, rt_allele.id) + assert len(rt_live) == 1 + assert rt_live[0].functional_consequence == "missense_variant" + # The authoritative allele's HGVS had no consequence, so it gets no row. + assert len(_live_consequences_for(session, authoritative_allele.id)) == 0 + + # One allele-keyed event per allele (never per-variant): the resolved RT allele is present, the + # unclassifiable authoritative allele is absent/no_record (queried, genuinely empty). + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "vep_functional_consequence" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" + + async def test_reference_identical_control_is_labelled( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """A wild-type control describes no sequence change, so VEP cannot resolve it. When the reference + port settles it as reference-identical, it is stored as a live consequence labelled + ``reference_identical`` (headline term ``no_change``) rather than dropped to ABSENT — so a control + can be told apart from an unannotated allele. The event is present/created: a result was reached.""" + _, allele = setup_sample_alleles_for_vep + hgvs = allele.hgvs_c or allele.hgvs_g + + reference_identical = ConsequenceResolution( + input=VepInput(hgvs=hgvs), + outcome=ConsequenceOutcome.RESOLVED, + consequence_terms=[NO_CHANGE_TERM], + most_severe_consequence=NO_CHANGE_TERM, + source=ConsequenceSource.REFERENCE_IDENTICAL, + ) + with patch(_RESOLVE, return_value={hgvs: reference_identical}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 + assert result.data["created_allele_count"] == 1 - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == NO_CHANGE_TERM + assert live[0].consequence_source == VepConsequenceSource.reference_identical + assert live[0].matched_transcript is None + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].allele_id == allele.id and events[0].variant_id is None - async def test_vep_none_consequence_routes_to_recoder( + async def test_resolves_in_chunks_and_merges_results( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_rt_derived_allele_for_vep, ): - """A None consequence from VEP Phase 1 is treated as a miss and routed through Recoder. + """Inputs are resolved in chunks (so progress can be reported between them), and the results from + every chunk are merged — not just the last chunk's. With the chunk size forced to one, each of the + two alleles resolves in its own call and both are still linked; a chunk whose result was dropped + would leave one allele unlinked.""" + _, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep - VEP can return an entry with most_severe_consequence=None when it recognises a variant - but cannot classify it. This should not silently fall into the UNKNOWN outcome branch — - instead it should be treated identically to an absent entry and sent to Recoder. - """ - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + def _resolve_each(inputs, **kwargs): + # One input per chunk here; each chunk returns only its own input's resolution, so a chunk + # whose result the job failed to merge would drop that allele's link. + return {inp.hgvs: _resolved(inp.hgvs, "missense_variant") for inp in inputs} with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {hgvs: None}, # VEP knows the variant but returns no consequence - {genomic_hgvs: "missense_variant"}, # Phase 3 on recoded genomic string - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), + patch("mavedb.worker.jobs.external_services.vep._RESOLUTION_CHUNK_SIZE", 1), + patch(_RESOLVE, side_effect=_resolve_each) as mock_resolve, ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 + # Two unique HGVS at chunk size one -> two resolution calls, each a single-input chunk. + assert mock_resolve.call_count == 2 + assert all(len(call.args[0]) == 1 for call in mock_resolve.call_args_list) + # Both alleles linked -> results from both chunks were merged, not overwritten by the last. + assert len(_live_consequences_for(session, authoritative_allele.id)) == 1 + assert len(_live_consequences_for(session, rt_allele.id)) == 1 + + async def test_skips_allele_already_at_current_release( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """An allele with a live consequence already at the current Ensembl release *and* resolver + version is skipped: no VEP query, the status reports SUCCESS/preexisting, and the row is not + churned.""" + _, allele = setup_sample_alleles_for_vep + + # Simulate a prior run at the current release and resolver version (current on both axes). + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + resolver_version=RESOLVER_VERSION, + access_date=date.today(), + ) + ) + session.commit() + + with patch(_RESOLVE) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_resolve.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + # Row not churned: still exactly one, still live. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None - async def test_most_severe_consequence_selected_from_multiple_genomic_hgvs( + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" + + async def test_stale_resolver_version_is_requeried( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """When Recoder returns multiple genomic strings with different consequences, the most severe wins.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_less_severe = "NC_000017.11:g.43094692C>T" # → missense_variant - genomic_more_severe = "NC_000017.11:g.43094692C>A" # → stop_gained - - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # Phase 1: VEP misses - { - genomic_less_severe: "missense_variant", - genomic_more_severe: "stop_gained", - }, # Phase 3: two different consequences - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_less_severe, genomic_more_severe]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele live at the *current* Ensembl release but resolved under an older resolver version is + NOT skipped — a resolution-rule fix (RESOLVER_VERSION bump) must re-query without waiting for an + Ensembl release bump. Re-resolving the same term reports preexisting and advances the resolver + version on the live row in place (no churn).""" + _, allele = setup_sample_alleles_for_vep + + # Current Ensembl release but a stale resolver version -> only the resolver axis is out of date. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + resolver_version=f"{RESOLVER_VERSION}-stale", + access_date=date.today() - timedelta(days=1), ) + ) + session.commit() - assert result.data["variants_with_consequences"] == 1 + with patch( + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_resolve.assert_called_once() # stale resolver version -> re-queried despite the current release + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "stop_gained" + # The live row is refreshed to the current resolver version (advanced in place, still one live row). + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].resolver_version == RESOLVER_VERSION - async def test_multiple_variants_sharing_hgvs_all_get_consequence( + async def test_new_release_same_consequence_bumps_in_place( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """All mapped variants that share an HGVS string each receive the consequence.""" - _, mapped_variant_1 = setup_sample_variants_for_vep - hgvs = mapped_variant_1.hgvs_assay_level - - score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) - variant_2 = Variant( - urn="urn:variant:test-variant-for-vep-2", - score_set_id=score_set.id, - hgvs_nt=hgvs, - data={"hgvs_c": hgvs}, + """An allele live at an *older* release is not skipped: the job re-queries it and reports it + preexisting (the unchanged consequence advances in place). The row-level in-place-bump semantics + are the linker's, covered by test_link_vep_unchanged_bumps_version_and_fills_provenance_in_place.""" + _, allele = setup_sample_alleles_for_vep + + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) ) - session.add(variant_2) session.commit() - mapped_variant_2 = MappedVariant( - variant_id=variant_2.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": hgvs, "syntax": "hgvs.c"}]}, - hgvs_assay_level=hgvs, + + with patch( + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_resolve.assert_called_once() # older release -> not skipped, re-queried + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + + async def test_force_requeries_unchanged_without_churn( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """force bypasses the current-release skip: an allele already at the current release, which would + normally be skipped, is re-queried instead. The no-churn-on-unchanged-value guarantee is the + linker's, covered by test_link_vep_unchanged_bumps_version_and_fills_provenance_in_place.""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence already current on both axes (would be skipped without force). + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + resolver_version=RESOLVER_VERSION, + access_date=date.today() - timedelta(days=1), + ) ) - session.add(mapped_variant_2) + session.commit() + + sample_populate_vep_run.job_params = {**sample_populate_vep_run.job_params, "force": True} session.commit() with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_resolve.assert_called_once() # force bypassed the current-release skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + + async def test_new_release_changed_consequence_supersedes( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """An allele live at an *older* release is not skipped: the job re-queries it and a changed + result is reported created. The supersede/retire row semantics are the linker's, covered by + test_link_vep_changed_consequence_supersedes.""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence at an older release -> not skipped, eligible for re-query. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() - assert result.data["variants_processed"] == 2 - assert result.data["variants_with_consequences"] == 2 + with patch( + _RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")} + ) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - session.refresh(mapped_variant_1) - session.refresh(mapped_variant_2) - assert mapped_variant_1.vep_functional_consequence == "missense_variant" - assert mapped_variant_2.vep_functional_consequence == "missense_variant" + mock_resolve.assert_called_once() # older release -> not skipped, re-queried + assert result.data["created_allele_count"] == 1 - async def test_vep_batch_api_exception_raises( + async def test_retained_on_absence_surfaced_in_outcome( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the VEP API propagates to the job management decorator.""" - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=RuntimeError("VEP API unreachable"), - ), - pytest.raises(RuntimeError, match="VEP API unreachable"), - ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele that had a consequence at an older release but for which VEP now returns nothing + keeps its prior value (not superseded) and is counted in ``retained_on_absence_count`` — the + job-metadata signal for a possible genuine disappearance.""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence at an older release -> re-queried, but VEP returns nothing this run. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() + + with patch(_RESOLVE, return_value={}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.status == JobStatus.SUCCEEDED + # Surfaced in the outcome metadata, and counted as preexisting (the allele still has a value). + assert result.data["retained_on_absence_count"] == 1 + assert result.data["preexisting_allele_count"] == 1 + assert result.data["absent_allele_count"] == 0 + + # The prior consequence was held in place, not superseded or dropped. + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "115" + + async def test_successful_linking_pipeline( + self, + session, + with_populated_domain_data, + mock_worker_ctx, + sample_populate_vep_run_pipeline, + sample_populate_vep_pipeline, + setup_sample_alleles_for_vep, + ): + """End-to-end successful linking within a pipeline updates both job and pipeline status.""" + _, allele = setup_sample_alleles_for_vep - async def test_variant_recoder_api_exception_raises( + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run_pipeline.id) + + assert result.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_exceptions_handled_by_decorators( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the Variant Recoder API propagates to the job management decorator.""" + """Exceptions during resolution are handled by the job decorators (Slack alert, ERRORED).""" with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - side_effect=RuntimeError("Recoder API unreachable"), - ), - pytest.raises(RuntimeError, match="Recoder API unreachable"), + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_send_slack_job_error.assert_called_once() + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.ERRORED + assert isinstance(result.exception, Exception) + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED @pytest.mark.asyncio @pytest.mark.integration -class TestPopulateVepForScoreSetIntegration: - """Integration tests for populate_vep_for_score_set run through an ARQ worker context.""" +class TestPopulateVepForScoreSetArqContext: + """Tests for the populate_vep_for_score_set job using the ARQ context fixture.""" - async def test_populate_vep_with_arq_context( + async def test_populate_vep_with_arq_context_independent( self, arq_redis, arq_worker, @@ -449,35 +685,29 @@ async def test_populate_vep_with_arq_context( with_populated_domain_data, with_populate_vep_job, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes successfully within an ARQ worker context.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level + """The VEP job links a consequence and records a SUCCESS annotation through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) await arq_worker.async_run() await arq_worker.run_check() - session.refresh(sample_populate_vep_run) - assert sample_populate_vep_run.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED - async def test_populate_vep_in_pipeline_context( + async def test_populate_vep_with_arq_context_pipeline( self, arq_redis, arq_worker, @@ -485,24 +715,73 @@ async def test_populate_vep_in_pipeline_context( with_populated_domain_data, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes and advances the pipeline when run in a pipeline context.""" - from mavedb.models.enums.job_pipeline import PipelineStatus - - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level + """The VEP job completes and advances the pipeline through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "synonymous_variant"}, - ): + with patch(_RESOLVE, return_value={allele.hgvs_c: _resolved(allele.hgvs_c, "missense_variant")}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() await arq_worker.run_check() + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED - session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_populate_vep_with_arq_context_exception_handling_independent( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + with_populate_vep_job, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job are handled with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED + + async def test_populate_vep_with_arq_context_exception_handling_pipeline( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + sample_populate_vep_pipeline, + sample_populate_vep_run_pipeline, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job fail the pipeline with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.ERRORED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/system/test_cleanup.py b/tests/worker/jobs/system/test_cleanup.py index 522b6167d..1a127326a 100644 --- a/tests/worker/jobs/system/test_cleanup.py +++ b/tests/worker/jobs/system/test_cleanup.py @@ -12,6 +12,7 @@ pytest.importorskip("arq") # Skip tests if arq is not installed +import asyncio from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, patch @@ -29,28 +30,67 @@ from mavedb.worker.jobs.system.cleanup import ( PENDING_TIMEOUT_MINUTES, PIPELINE_STUCK_TIMEOUT_MINUTES, + PROGRESS_STALL_MINUTES, RUNNING_TIMEOUT_MINUTES, cleanup_stalled_jobs, ) from mavedb.worker.lib.managers.job_manager import JobManager -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import FAST_LANE_QUEUE_NAME, arq_job_id pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @pytest.fixture def mock_arq_job_not_found(): - """Mock ArqJob.status() to return not_found, simulating a crashed-enqueue QUEUED job.""" + """Mock ArqJob for a crashed-enqueue QUEUED job: status() is not_found, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) yield mock_arq_job @pytest.fixture def mock_arq_job_in_redis(): - """Mock ArqJob.status() to return queued, simulating a legitimately queued job in ARQ Redis.""" + """Mock ArqJob for a legitimately queued job: status() is queued, and abort() confirms dead.""" with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.queued) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_confirmed(): + """Mock ArqJob.abort() to return True — the stalled RUNNING attempt was alive and is now dead. + + This is the signal that makes it safe for the sweeper to re-drive a stalled RUNNING job. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=True) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_absent(): + """Mock ArqJob.abort() to return False — the job is absent from ARQ (crashed / never enqueued). + + Nothing is running, so it is still safe to re-drive. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.not_found) + mock_arq_job.return_value.abort = AsyncMock(return_value=False) + yield mock_arq_job + + +@pytest.fixture +def mock_arq_abort_wedged(): + """Mock ArqJob.abort() to time out — the coroutine is wedged in a blocking call and won't die. + + The sweeper must NOT re-drive such a job (it could still be running), to avoid a double-run. + """ + with patch("mavedb.worker.jobs.system.cleanup.ArqJob") as mock_arq_job: + mock_arq_job.return_value.status = AsyncMock(return_value=ArqJobStatus.in_progress) + mock_arq_job.return_value.abort = AsyncMock(side_effect=asyncio.TimeoutError()) yield mock_arq_job @@ -148,7 +188,7 @@ async def test_cleanup_stalled_queued_job_max_retries_reached( assert "stalled" in stalled_job.error_message.lower() async def test_cleanup_stalled_running_job_with_retries( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with retries remaining.""" # Create a stalled RUNNING job in the database @@ -184,7 +224,7 @@ async def test_cleanup_stalled_running_job_with_retries( assert stalled_job.finished_at is None async def test_cleanup_stalled_running_job_max_retries_reached( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test cleanup of a stalled RUNNING job with max retries reached.""" # Create a stalled RUNNING job with max retries @@ -257,6 +297,115 @@ async def test_cleanup_stalled_running_job_missing_started_at( assert stalled_job.status == JobStatus.RUNNING assert stalled_job.retry_count == 0 + async def test_cleanup_detects_running_job_by_stale_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed + ): + """A RUNNING job with a recent started_at but a stale progress heartbeat is detected and re-driven. + + This is the primary staleness signal: staleness is judged by progress_updated_at, not total runtime. + """ + stalled_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), # well within the wall-clock backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(stalled_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort of the wedged attempt was confirmed, so the job was re-driven. + mock_arq_abort_confirmed.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_called_once() + + assert result.data["total_cleaned"] == 1 + assert stalled_job.urn in result.data["running_jobs"] + + session.refresh(stalled_job) + assert stalled_job.status == JobStatus.QUEUED + assert stalled_job.retry_count == 1 + + async def test_cleanup_leaves_running_job_with_fresh_heartbeat( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + ): + """A RUNNING job that has been running a long time but is still checkpointing progress is left alone. + + This is the VEP fan-out case: legitimate multi-hour runtime, well past the old 150-min limit, + but under the wall-clock backstop and with a fresh heartbeat — must NOT be treated as stalled. + """ + healthy_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=310), + started_at=datetime.now(timezone.utc) - timedelta(minutes=300), # 5h — past old limit, under backstop + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=2), # fresh heartbeat + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(healthy_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + assert result.data["total_cleaned"] == 0 + assert healthy_job.urn not in result.data["running_jobs"] + + session.refresh(healthy_job) + assert healthy_job.status == JobStatus.RUNNING + assert healthy_job.retry_count == 0 + + async def test_cleanup_does_not_redrive_wedged_running_job( + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_wedged + ): + """A stalled RUNNING job whose abort does not confirm is left in place, not re-driven. + + Re-driving a job that may still be executing (wedged in a blocking sync call) would recreate + the historical double-run, so the sweeper takes no action and leaves it for a later sweep. + """ + wedged_job = JobRun( + job_type="test_job", + job_function="test_function", + status=JobStatus.RUNNING, + created_at=datetime.now(timezone.utc) - timedelta(minutes=40), + started_at=datetime.now(timezone.utc) - timedelta(minutes=30), + progress_updated_at=datetime.now(timezone.utc) - timedelta(minutes=PROGRESS_STALL_MINUTES + 5), + finished_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(wedged_job) + session.commit() + + result = await cleanup_stalled_jobs( + mock_worker_ctx, None, JobManager(session, mock_worker_ctx["redis"], sample_cleanup_job_run.id) + ) + + # Abort was attempted but never confirmed, so no retry was enqueued. + mock_arq_abort_wedged.return_value.abort.assert_awaited_once() + mock_worker_ctx["redis"].enqueue_job.assert_not_called() + + assert result.data["total_cleaned"] == 0 + assert wedged_job.urn not in result.data["running_jobs"] + + session.refresh(wedged_job) + assert wedged_job.status == JobStatus.RUNNING + assert wedged_job.retry_count == 0 + async def test_cleanup_stalled_pending_job_with_retries( self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job ): @@ -477,7 +626,7 @@ async def test_cleanup_stalled_queued_standalone_job_enqueue_failure( assert "Failed to enqueue after stall recovery" in stalled_job.error_message async def test_cleanup_stalled_running_standalone_job_enqueue_failure( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled standalone RUNNING job is marked FAILED if ARQ enqueue fails.""" @@ -559,7 +708,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_satisfied( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_satisfied( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with satisfied dependencies is enqueued.""" # Create a pipeline with all dependencies satisfied @@ -735,7 +884,7 @@ async def test_cleanup_stalled_queued_pipeline_job_dependencies_not_ready( assert stalled_job.retry_count == 1 async def test_cleanup_stalled_running_pipeline_job_dependencies_failed( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with failed dependencies is skipped.""" # Create a pipeline @@ -867,7 +1016,7 @@ async def test_cleanup_stalled_pending_pipeline_job_dependencies_failed( assert stalled_job.retry_count == 0 async def test_cleanup_stalled_running_pipeline_job_dependencies_not_ready( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that stalled pipeline RUNNING job with dependencies not ready is skipped.""" # Create a pipeline @@ -1249,7 +1398,7 @@ async def test_cleanup_sends_slack_when_max_retries_reached_queued_job( assert call_kwargs["max_retries"] == 3 async def test_cleanup_sends_slack_when_max_retries_reached_running_job( - self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job + self, session, mock_worker_ctx, sample_cleanup_job_run, with_cleanup_job, mock_arq_abort_confirmed ): """Test that send_slack_job_failure is called when sweeper permanently fails a stalled RUNNING job.""" stalled_job = JobRun( @@ -1565,6 +1714,37 @@ async def test_cleanup_integration_excludes_recent_jobs(self, standalone_worker_ assert recent_queued.retry_count == 0 assert recent_running.retry_count == 0 + async def test_cleanup_integration_leaves_fast_lane_queued_job_alone(self, standalone_worker_context, session): + """Integration test: the Redis presence check looks for a fast-lane job on the fast-lane queue.""" + queued_creation = JobRun( + job_type="test_job", + job_function="create_variants_for_score_set", + status=JobStatus.QUEUED, + created_at=datetime.now(timezone.utc) - timedelta(minutes=PENDING_TIMEOUT_MINUTES + 5), + started_at=None, + max_retries=3, + retry_count=0, + job_params={}, + ) + session.add(queued_creation) + session.commit() + + arq_redis = standalone_worker_context["redis"] + await arq_redis.enqueue_job( + "create_variants_for_score_set", + queued_creation.id, + _job_id=arq_job_id(queued_creation), + _queue_name=FAST_LANE_QUEUE_NAME, + ) + + result = await cleanup_stalled_jobs(standalone_worker_context) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["total_cleaned"] == 0 + session.refresh(queued_creation) + assert queued_creation.status == JobStatus.QUEUED + assert queued_creation.retry_count == 0 + async def test_cleanup_integration_updates_progress_correctly(self, standalone_worker_context, session): """Integration test: cleanup job updates progress correctly and returns proper data.""" # Create stalled jobs to trigger progress updates across different states @@ -2490,14 +2670,16 @@ async def test_cleanup_arq_integration(self, arq_redis, arq_worker, standalone_w await arq_worker.async_run() # Don't call run_check() - the retried test_function doesn't exist and would fail - # Verify the cleanup job succeeded - cleanup_job = session.execute( - select(JobRun).where(JobRun.job_function == "cleanup_stalled_jobs") - ).scalar_one_or_none() + # Verify the cleanup job succeeded. Asserted over every run rather than via + # `scalar_one_or_none`, because the `arq_worker` fixture registers `BACKGROUND_CRONJOBS` and + # `cleanup_stalled_jobs_cron` fires at minutes {5, 15, 25, 35, 45, 55}. A burst run that happens + # to straddle one of those minutes executes the cron alongside the job enqueued above, so the + # row count depends on the wall clock. Every run must still succeed, which is the property here. + cleanup_jobs = session.scalars(select(JobRun).where(JobRun.job_function == "cleanup_stalled_jobs")).all() - assert cleanup_job is not None - assert cleanup_job.status == JobStatus.SUCCEEDED - assert cleanup_job.job_type == "cron_job" + assert cleanup_jobs + assert all(job.status == JobStatus.SUCCEEDED for job in cleanup_jobs) + assert all(job.job_type == "cron_job" for job in cleanup_jobs) # Verify the stalled job was cleaned up session.refresh(stalled_job) diff --git a/tests/worker/jobs/test_allele_translations.py b/tests/worker/jobs/test_allele_translations.py new file mode 100644 index 000000000..0e1c1b12c --- /dev/null +++ b/tests/worker/jobs/test_allele_translations.py @@ -0,0 +1,41 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from datetime import datetime, timezone + +from mavedb.lib.alleles import get_allele_translations + + +@pytest.mark.unit +class TestGetAlleleTranslations: + """The cross-layer equivalence query traverses the MappingRecordAllele link graph. + + Uses the RT-derived fixture, which links an authoritative allele and a derived (cross-layer) + allele to one MappingRecord — exactly the co-membership the query resolves. + """ + + def test_returns_full_equivalence_set_from_any_member(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + from_authoritative = {a.id for a in get_allele_translations(session, authoritative_allele.id)} + from_rt = {a.id for a in get_allele_translations(session, rt_allele.id)} + + # Reachable from either member, and includes the anchor itself — the full equivalence set. + assert from_authoritative == {authoritative_allele.id, rt_allele.id} + assert from_rt == {authoritative_allele.id, rt_allele.id} + + def test_as_of_before_links_existed_is_empty(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, _rt = setup_rt_derived_allele_with_caid + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + assert get_allele_translations(session, authoritative_allele.id, as_of=past) == [] + + def test_as_of_after_links_existed_returns_set(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + ids = {a.id for a in get_allele_translations(session, authoritative_allele.id, as_of=future)} + assert ids == {authoritative_allele.id, rt_allele.id} diff --git a/tests/worker/jobs/variant_processing/test_creation.py b/tests/worker/jobs/variant_processing/test_creation.py index d0d37562a..5bff5b1a5 100644 --- a/tests/worker/jobs/variant_processing/test_creation.py +++ b/tests/worker/jobs/variant_processing/test_creation.py @@ -5,13 +5,23 @@ pytest.importorskip("arq") import math -from unittest.mock import ANY, MagicMock, call, patch - +from datetime import date, timedelta +from unittest.mock import ANY, call, patch + +from mavedb.lib.mondo import get_generic_disease_term +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.calibration_control import CalibrationControl +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.calibration_control_status import CalibrationControlStatus +from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord from mavedb.models.pipeline import Pipeline +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.variant import Variant from mavedb.worker.jobs.variant_processing.creation import create_variants_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager @@ -69,7 +79,7 @@ async def test_create_variants_for_score_set_calls_s3_client_with_correct_parame ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -153,7 +163,7 @@ async def test_create_variants_for_score_set_counts_file_can_be_optional( ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -270,7 +280,7 @@ async def test_create_variants_for_score_set_removes_existing_variants_before_cr ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -286,6 +296,80 @@ async def test_create_variants_for_score_set_removes_existing_variants_before_cr session.refresh(sample_score_set) assert sample_score_set.num_variants == 0 # Updated after creation + async def test_create_variants_for_score_set_removes_existing_mapped_variants_before_creation( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + create_variants_sample_params, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_independent_variant_creation_run, + ): + # An existing variant that has been mapped: its mapping record and annotation event must not + # block its removal. + sample_score_set.num_variants = 1 + variant = Variant(data={}, score_set_id=sample_score_set.id) + session.add(variant) + session.commit() + session.add( + MappingRecord( + variant_id=variant.id, + score_set_id=sample_score_set.id, + assay_level="cdna", + mapping_api_version="test.0.0", + ) + ) + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + ) + ) + session.commit() + variant_id = variant.id + + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + # Mock pd.read_csv to return sample dataframes + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[sample_score_dataframe, sample_count_dataframe], + ), + patch( + "mavedb.worker.jobs.variant_processing.creation.validate_and_standardize_dataframe_pair", + return_value=( + sample_score_dataframe, + sample_count_dataframe, + create_variants_sample_params["score_columns_metadata"], + create_variants_sample_params["count_columns_metadata"], + ), + ), + patch( + "mavedb.worker.jobs.variant_processing.creation.create_variants_data", + return_value=[{"data": {"score_data": {"score": 1.0}}}], + ), + patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), + ): + await create_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_creation_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_creation_run.id), + ) + + # Verify that existing variants have been removed + remaining_variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() + assert len(remaining_variants) == 0 + assert session.query(MappingRecord).filter(MappingRecord.score_set_id == sample_score_set.id).count() == 0 + assert session.query(AnnotationEvent).filter(AnnotationEvent.variant_id == variant_id).count() == 0 + session.refresh(sample_score_set) + assert sample_score_set.num_variants == 0 # Updated after creation + async def test_create_variants_for_score_set_updates_processing_state( self, session, @@ -317,7 +401,7 @@ async def test_create_variants_for_score_set_updates_processing_state( ), patch( "mavedb.worker.jobs.variant_processing.creation.create_variants_data", - return_value=[MagicMock(spec=Variant)], + return_value=[{"data": {"score_data": {"score": 1.0}}}], ), patch("mavedb.worker.jobs.variant_processing.creation.create_variants", return_value=None), ): @@ -462,6 +546,12 @@ async def test_create_variants_for_score_set_independent_job( assert sample_score_set.processing_state == ProcessingState.success assert sample_score_set.mapping_state == MappingState.queued + distribution = sample_score_set.score_distribution + assert distribution is not None + assert sum(distribution["counts"]) + distribution["null_count"] == len(created_variants) + assert distribution["min"] == sample_score_dataframe["score"].min() + assert distribution["max"] == sample_score_dataframe["score"].max() + # Verify that the created variants have expected data for variant in created_variants: assert variant.data # Ensure data is not empty @@ -619,6 +709,10 @@ async def test_create_variants_for_score_set_replaces_variants( replaced_variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() assert len(replaced_variants) == sample_score_dataframe.shape[0] + session.refresh(sample_score_set) + assert sample_score_set.score_distribution["min"] == updated_score_dataframe["score"].min() + assert sample_score_set.score_distribution["max"] == updated_score_dataframe["score"].max() + # Verify that the variants have been replaced with updated data for variant in replaced_variants: assert variant.data # Ensure data is not empty @@ -870,6 +964,55 @@ async def test_create_variants_for_score_set_generic_exception_handling_during_r ) assert job_run.status == JobStatus.ERRORED + async def test_create_variants_for_score_set_keeps_score_distribution_when_creation_fails( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_independent_variant_creation_run, + ): + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[sample_score_dataframe, sample_count_dataframe], + ), + ): + await create_variants_for_score_set(mock_worker_ctx, sample_independent_variant_creation_run.id) + + session.refresh(sample_score_set) + initial_distribution = sample_score_set.score_distribution + assert initial_distribution is not None + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + # The new scores change the summary, which is assigned before the variant insert fails. + updated_score_dataframe = sample_score_dataframe.copy() + updated_score_dataframe["score"] += 10 + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[updated_score_dataframe, sample_count_dataframe], + ), + patch( + "mavedb.worker.jobs.variant_processing.creation.create_variants", + side_effect=Exception("Generic exception during variant creation"), + ), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error"), + ): + await create_variants_for_score_set(mock_worker_ctx, sample_independent_variant_creation_run.id) + + session.refresh(sample_score_set) + assert sample_score_set.processing_state == ProcessingState.failed + assert sample_score_set.score_distribution == initial_distribution + ## Pipeline failure workflow async def test_create_variants_for_score_set_pipeline_job_generic_exception_handling( @@ -1188,3 +1331,456 @@ async def test_create_variants_for_score_set_with_arq_context_generic_exception_ ) for run in other_runs: assert run.status == JobStatus.SKIPPED + + +def _make_calibration(session, score_set, user, controls_not_phi=None): + """Persist a bare calibration on a score set; controls and bin membership need a parent.""" + calibration = ScoreCalibration( + title="Relink test calibration", + score_set_id=score_set.id, + disease_term=get_generic_disease_term(session), + controls_not_phi=controls_not_phi, + created_by=user, + modified_by=user, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + return calibration + + +def _variants_by_hgvs_nt(session, score_set): + return {variant.hgvs_nt: variant for variant in session.query(Variant).filter_by(score_set_id=score_set.id).all()} + + +def _without_variant(scores_df, counts_df, hgvs_nt): + """Return the score/count pair with one variant removed, simulating a corrected re-upload.""" + return ( + scores_df[scores_df["hgvs_nt"] != hgvs_nt].copy(), + counts_df[counts_df["hgvs_nt"] != hgvs_nt].copy(), + ) + + +async def _run_creation_job(mock_worker_ctx, mock_s3_client, job_run, scores_df, counts_df): + with ( + patch.object(mock_s3_client, "download_fileobj", return_value=None), + patch( + "mavedb.worker.jobs.variant_processing.creation.pd.read_csv", + side_effect=[scores_df, counts_df], + ), + ): + await create_variants_for_score_set(mock_worker_ctx, job_run.id) + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestCreateVariantsForScoreSetCalibrationRelinking: + """Re-upload behavior for score sets whose calibrations reference variants. + + Calibration controls and functional classification bin membership hold plain foreign keys into + ``variants``, which a re-upload deletes and recreates wholesale. These tests pin both halves of + the fix: the delete no longer trips those constraints, and surviving references are re-resolved + against the new rows by HGVS identity rather than by the positional variant URN. + """ + + async def test_reupload_with_calibration_controls_does_not_violate_foreign_key( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + calibration = _make_calibration(session, sample_score_set, sample_user) + session.add( + CalibrationControl( + calibration_id=calibration.id, + variant_id=_variants_by_hgvs_nt(session, sample_score_set)["c.1A>T"].id, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=sample_user, + modified_by=sample_user, + ) + ) + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + # Regression: this second upload raised ForeignKeyViolation on calibration_controls_variant_id_fkey. + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + session.refresh(sample_score_set) + assert sample_score_set.processing_state == ProcessingState.success + assert session.query(CalibrationControl).filter_by(calibration_id=calibration.id).count() == 1 + + async def test_controls_relink_to_new_variants_when_every_identity_persists( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_extra_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + original_variant_ids = { + hgvs: variant.id for hgvs, variant in _variants_by_hgvs_nt(session, sample_score_set).items() + } + calibration = _make_calibration(session, sample_score_set, sample_user, controls_not_phi=True) + + # An older submission date proves the relinked row keeps its own provenance rather than + # inheriting the re-upload's. + submitted_on = date.today() - timedelta(days=30) + for hgvs_nt, status in ( + ("c.1A>T", CalibrationControlStatus.pathogenic), + ("c.2T>A", CalibrationControlStatus.benign), + ): + session.add( + CalibrationControl( + calibration_id=calibration.id, + variant_id=original_variant_ids[hgvs_nt], + clinical_status=status, + created_by=sample_extra_user, + modified_by=sample_extra_user, + creation_date=submitted_on, + modification_date=submitted_on, + ) + ) + + updated_scores = sample_score_dataframe.copy() + updated_scores["score"] += 10 + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + updated_scores, + sample_count_dataframe.copy(), + ) + + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert set(new_variants) == set(original_variant_ids) + # Every variant row really was replaced, so a stale link would be visible below. + assert all(new_variants[hgvs].id != variant_id for hgvs, variant_id in original_variant_ids.items()) + + controls = session.query(CalibrationControl).filter_by(calibration_id=calibration.id).all() + assert len(controls) == 2 + assert {control.variant_id for control in controls} == { + new_variants["c.1A>T"].id, + new_variants["c.2T>A"].id, + } + assert {control.clinical_status for control in controls} == { + CalibrationControlStatus.pathogenic, + CalibrationControlStatus.benign, + } + assert all(control.created_by_id == sample_extra_user.id for control in controls) + assert all(control.creation_date == submitted_on for control in controls) + assert all(control.modified_by_id == sample_user.id for control in controls) + + # Nothing was dropped, so the submitter's PHI affirmation still describes this control set. + session.refresh(calibration) + assert calibration.controls_not_phi is True + + async def test_controls_absent_from_new_upload_are_dropped_and_keep_phi_affirmation( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + original_variants = _variants_by_hgvs_nt(session, sample_score_set) + calibration = _make_calibration(session, sample_score_set, sample_user, controls_not_phi=True) + for hgvs_nt in ("c.1A>T", "c.4C>G"): + session.add( + CalibrationControl( + calibration_id=calibration.id, + variant_id=original_variants[hgvs_nt].id, + clinical_status=CalibrationControlStatus.pathogenic, + created_by=sample_user, + modified_by=sample_user, + ) + ) + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + trimmed_scores, trimmed_counts = _without_variant(sample_score_dataframe, sample_count_dataframe, "c.4C>G") + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + trimmed_scores, + trimmed_counts, + ) + + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert "c.4C>G" not in new_variants + + controls = session.query(CalibrationControl).filter_by(calibration_id=calibration.id).all() + assert len(controls) == 1 + assert controls[0].variant_id == new_variants["c.1A>T"].id + + # The surviving controls are a subset of the affirmed set, so the affirmation still covers them. + session.refresh(calibration) + assert calibration.controls_not_phi is True + + assert recorded_context["calibration_controls_relinked"] == 1 + assert recorded_context["calibration_controls_dropped"] == 1 + + async def test_range_based_classification_membership_is_rebinned_against_new_scores( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + # Sample scores: c.1A>T 0.3, c.2T>A 0.0, c.3G>C -1.65, c.4C>G unscored. + original_variants = _variants_by_hgvs_nt(session, sample_score_set) + calibration = _make_calibration(session, sample_score_set, sample_user) + normal = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="Normal function", + range=[0.0, None], + inclusive_lower_bound=True, + ) + normal.variants = [original_variants["c.1A>T"], original_variants["c.2T>A"]] + abnormal = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="Abnormal function", + range=[None, 0.0], + inclusive_upper_bound=False, + ) + abnormal.variants = [original_variants["c.3G>C"]] + session.add_all([normal, abnormal]) + + original_variant_ids = {variant.id for variant in original_variants.values()} + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + normal_id, abnormal_id = normal.id, abnormal.id + + # Shifting every score up by two moves c.3G>C across the threshold, so recorded membership and + # recomputed membership disagree and the test can tell which one was applied. + updated_scores = sample_score_dataframe.copy() + updated_scores["score"] += 2 + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + updated_scores, + sample_count_dataframe.copy(), + ) + + session.expire_all() + normal = session.get(ScoreCalibrationFunctionalClassification, normal_id) + abnormal = session.get(ScoreCalibrationFunctionalClassification, abnormal_id) + + # Membership follows the new scores rather than what was recorded: c.3G>C (-1.65 -> 0.35) moves + # into the normal bin, emptying the abnormal one. c.4C>G stays out of both, having no score. + assert {variant.hgvs_nt for variant in normal.variants} == {"c.1A>T", "c.2T>A", "c.3G>C"} + assert normal.variant_count == 3 + assert abnormal.variants == [] + assert abnormal.variant_count == 0 + + # Re-binned membership points at the new variant rows, not the deleted ones. + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert {variant.id for variant in normal.variants} <= {variant.id for variant in new_variants.values()} + assert {variant.id for variant in normal.variants}.isdisjoint(original_variant_ids) + + assert recorded_context["calibration_classifications_rebinned"] == 2 + assert recorded_context["calibration_classification_members_rebinned"] == 3 + # Range bins are recomputed, never relinked, so nothing is reported as carried across. + assert recorded_context["calibration_classification_members_relinked"] == 0 + + async def test_class_based_classification_membership_relinks_by_hgvs_identity( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_user, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + original_variants = _variants_by_hgvs_nt(session, sample_score_set) + calibration = _make_calibration(session, sample_score_set, sample_user) + + # A class-based bin comes from an uploaded classes file that a score upload does not carry, so + # its membership can only survive by identity. + classification = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="Loss of function", + class_="loss_of_function", + ) + classification.variants = [original_variants["c.1A>T"], original_variants["c.4C>G"]] + session.add(classification) + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + classification_id = classification.id + + trimmed_scores, trimmed_counts = _without_variant(sample_score_dataframe, sample_count_dataframe, "c.4C>G") + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + trimmed_scores, + trimmed_counts, + ) + + new_variants = _variants_by_hgvs_nt(session, sample_score_set) + assert "c.4C>G" not in new_variants + + session.expire_all() + classification = session.get(ScoreCalibrationFunctionalClassification, classification_id) + + # The surviving member is relinked to its new row; the one absent from the upload is dropped. + assert [variant.id for variant in classification.variants] == [new_variants["c.1A>T"].id] + assert classification.variant_count == 1 + + assert recorded_context["calibration_classification_members_relinked"] == 1 + assert recorded_context["calibration_classification_members_dropped"] == 1 + assert recorded_context["calibration_classifications_rebinned"] == 0 + + async def test_reupload_without_calibrations_is_unaffected( + self, + session, + with_independent_processing_runs, + with_populated_domain_data, + mock_worker_ctx, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + sample_score_set, + sample_independent_variant_creation_run, + ): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + sample_independent_variant_creation_run.status = JobStatus.PENDING + session.commit() + + recorded_context: dict = {} + original_save_to_context = JobManager.save_to_context + + def _capture_context(self, ctx): + recorded_context.update(ctx) + return original_save_to_context(self, ctx) + + with patch.object(JobManager, "save_to_context", _capture_context): + await _run_creation_job( + mock_worker_ctx, + mock_s3_client, + sample_independent_variant_creation_run, + sample_score_dataframe, + sample_count_dataframe, + ) + + session.refresh(sample_score_set) + assert sample_score_set.processing_state == ProcessingState.success + assert sample_score_set.num_variants == sample_score_dataframe.shape[0] + # The relink path stays out of the way entirely when there is nothing to relink. + assert "calibration_controls_relinked" not in recorded_context diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index dc2ac843e..ca13d8505 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -4,19 +4,32 @@ pytest.importorskip("arq") +import asyncio +import contextlib from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import date from unittest.mock import MagicMock, patch from sqlalchemy.exc import NoResultFound from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.worker.jobs.variant_processing.mapping import ( + MAP_BUDGET_BASE_SECONDS, + MAP_BUDGET_MAX_SECONDS, + _keepalive_progress, + _map_budget_seconds, + map_variants_for_score_set, +) from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import TEST_CODING_LAYER, TEST_GENOMIC_LAYER, TEST_PROTEIN_LAYER from tests.helpers.util.setup.worker import construct_mock_mapping_output, create_variants_in_score_set @@ -24,6 +37,25 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def authoritative_allele_for(session, mapping_record): + """Return the authoritative (post-mapped) Allele linked to a mapping record, or None. + + Under the allele data model the post-mapped VRS representation lives on an + ``Allele`` joined to the ``MappingRecord`` through ``MappingRecordAllele``. + A successfully post-mapped variant has exactly one authoritative link; a + variant that failed to post-map has a mapping record but no such link. + """ + return ( + session.query(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .filter( + MappingRecordAllele.mapping_record_id == mapping_record.id, + MappingRecordAllele.is_authoritative.is_(True), + ) + .one_or_none() + ) + + @pytest.mark.unit @pytest.mark.asyncio class TestMapVariantsForScoreSetUnit: @@ -66,8 +98,8 @@ async def test_map_variants_for_score_set_no_mapping_results( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -90,7 +122,10 @@ async def test_map_variants_for_score_set_no_mapped_scores( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [], "error_message": "No variants were mapped for this score set"} + { + "mapped_scores": [], + "error_message": "No variants were mapped for this score set", + } ), ), ): @@ -110,8 +145,8 @@ async def test_map_variants_for_score_set_no_mapped_scores( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -134,7 +169,10 @@ async def test_map_variants_for_score_set_no_reference_data( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [MagicMock()], "error_message": "Reference metadata missing from mapping results"} + { + "mapped_scores": [MagicMock()], + "error_message": "Reference metadata missing from mapping results", + } ), ), ): @@ -154,8 +192,8 @@ async def test_map_variants_for_score_set_no_reference_data( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -209,8 +247,8 @@ async def test_map_variants_for_score_set_nonexistent_target_gene( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -256,8 +294,8 @@ async def test_map_variants_for_score_set_returns_variants_not_in_score_set( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -318,20 +356,132 @@ async def dummy_mapping_job(): for target in sample_score_set.target_genes: assert target.mapped_hgnc_name is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" + + async def test_map_variants_for_score_set_clears_stale_uniprot_id( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A remap must clear any UniProt ID left over from a prior mapping run. + + The downstream UniProt job only writes on success, so a stale value would otherwise + persist and be mismatched against the newly mapped metadata. + """ + + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=True, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + + variant = Variant( + score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={} + ) + session.add(variant) + + for target in sample_score_set.target_genes: + target.uniprot_id_from_mapped_metadata = "P00000" + session.add(target) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + + for target in sample_score_set.target_genes: + assert target.uniprot_id_from_mapped_metadata is None + + async def test_map_variants_for_score_set_clears_stale_mapped_hgnc_name( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A remap must clear any mapped HGNC name left over from a prior mapping run. + + Gene-level info is only written when the mapping results include it, so a stale value + would otherwise persist and be mismatched against the newly mapped metadata. + """ + + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=False, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + + variant = Variant( + score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={} + ) + session.add(variant) + + for target in sample_score_set.target_genes: + target.mapped_hgnc_name = "STALE1" + session.add(target) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + + for target in sample_score_set.target_genes: + assert target.mapped_hgnc_name is None @pytest.mark.parametrize( "with_layers", @@ -435,20 +585,64 @@ async def dummy_mapping_job(): else: assert target.post_mapped_metadata.get("protein") is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" + + async def test_persists_cdna_target_gene_mapping_with_reference_accession_and_null_qc( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """An identity cdna TargetGeneMapping (a cdna layer with no per-variant scores + joining it) persists with its reference_accession (NM_) and null QC/counts -- the + artifact reverse translation consumes to resolve the projection transcript.""" + variant = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + + # Genomic assay layer + an identity cdna layer (no cdna per-variant scores). + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, score_set=sample_score_set, with_layers={"g", "c"} + ) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + assert result.status == JobStatus.SUCCEEDED + + cdna_tgms = ( + session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == SequenceLevel.cdna).all() + ) + assert len(cdna_tgms) == len(sample_score_set.target_genes) + tgm = cdna_tgms[0] + assert tgm.reference_accession == "NM_999999.1" + # Identity row: no mapped_variant joins it, so QC/counts are null. + assert tgm.total_variants is None + assert tgm.variants_mapped_cleanly is None + assert tgm.percent_identity is None async def test_map_variants_for_score_set_success_no_successful_mapping( self, @@ -501,24 +695,23 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.failed assert sample_score_set.mapping_errors["error_message"] == "All variants failed to map." - # Verify that one mapped variant was created. Although no successful mapping, an entry is still created. - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that one mapping record was created. Although no successful mapping, an entry is still created. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 - # Verify that the mapped variant has no post-mapped data - mapped_variant = mapped_variants[0] - assert mapped_variant.post_mapped == {} + # Verify that the mapping record has no authoritative (post-mapped) allele + assert authoritative_allele_for(session, mapping_records[0]) is None # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "failed" + assert annotation_statuses[0].disposition == "failed" async def test_map_variants_for_score_set_incomplete_mapping( self, @@ -551,14 +744,14 @@ async def dummy_mapping_job(): hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", ) variant2 = Variant( score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, - urn="variant:2", + urn=f"{sample_score_set.urn}#12", ) session.add_all([variant1, variant2]) session.commit() @@ -584,38 +777,124 @@ async def dummy_mapping_job(): # Although only one variant was successfully mapped, verify that an entity was created # for each variant in the score set - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 - # Verify that only one variant has post-mapped data - mapped_variant_with_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped != {}).one_or_none() - ) - assert mapped_variant_with_post_data is not None - - mapped_variant_without_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped == {}).one_or_none() - ) - assert mapped_variant_without_post_data is not None + # Verify that exactly one mapping record has post-mapped (authoritative) allele data + records_with_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is not None] + records_without_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is None] + assert len(records_with_post_data) == 1 + assert len(records_without_post_data) == 1 # Verify that annotation statuses were created and correct annotation_status_success = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "success") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "present") .all() ) assert len(annotation_status_success) == 1 assert annotation_status_success[0].annotation_type == "vrs_mapping" annotation_status_failed = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "failed") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "failed") .all() ) assert len(annotation_status_failed) == 1 assert annotation_status_failed[0].annotation_type == "vrs_mapping" + async def test_map_variants_for_score_set_benign_outcomes_are_not_failures( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A score set whose only unmapped variants are benign absences (intronic / no + protein consequence) is ``complete``, not ``incomplete``/``failed``: benign + outcomes carry no allele, so they are recorded as ``absent`` (an informative + biological negative), never ``failed``.""" + + async def dummy_mapping_job(): + mapping_output = await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=True, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + # Re-stamp the emitted records as benign absences: no allele, no failure. The + # helper only produces MAPPED/FAILED, so we override here to exercise the path. + benign_outcomes = ["intronic", "no_protein_consequence"] + for idx, mapped_score in enumerate(mapping_output["mapped_scores"]): + mapped_score["pre_mapped"] = {} + mapped_score["post_mapped"] = {} + mapped_score["outcome"] = benign_outcomes[idx % len(benign_outcomes)] + return mapping_output + + variant1 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + urn=f"{sample_score_set.urn}#11", + ) + variant2 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + urn=f"{sample_score_set.urn}#12", + ) + session.add_all([variant1, variant2]) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + # The run succeeded and the score set is complete -- nothing genuinely failed. + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data["mapped_count"] == 0 + assert result.data["failed_count"] == 0 + assert result.data["skipped_count"] == 2 + assert sample_score_set.mapping_state == MappingState.complete + assert sample_score_set.mapping_errors is None + + # A record exists per variant, but with no authoritative allele. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 + assert all(authoritative_allele_for(session, r) is None for r in mapping_records) + + # Benign outcomes carry no allele → recorded as `absent` (not `failed`), with the finer + # outcome preserved as the event `reason`. + events = ( + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id) + .all() + ) + assert len(events) == 2 + assert all(e.disposition == "absent" for e in events) + recorded_outcomes = {e.reason for e in events} + assert recorded_outcomes == {"intronic", "no_protein_consequence"} + async def test_map_variants_for_score_set_complete_mapping( self, session, @@ -647,14 +926,14 @@ async def dummy_mapping_job(): hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}, - urn="variant:1", + urn=f"{sample_score_set.urn}#11", ) variant2 = Variant( score_set_id=sample_score_set.id, hgvs_nt="NM_000000.1:c.2G>T", hgvs_pro="NP_000000.1:p.Val2Leu", data={}, - urn="variant:2", + urn=f"{sample_score_set.urn}#12", ) session.add_all([variant1, variant2]) session.commit() @@ -678,29 +957,29 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 # Verify that both variants have post-mapped data. I'm comfortable assuming the # data is correct given our layer permutation tests above. - for urn in ["variant:1", "variant:2"]: - mapped_variant = session.query(MappedVariant).filter(MappedVariant.variant.has(urn=urn)).one_or_none() - assert mapped_variant is not None - assert mapped_variant.post_mapped != {} - assert mapped_variant.hgvs_assay_level is not None + for urn in [f"{sample_score_set.urn}#11", f"{sample_score_set.urn}#12"]: + mapping_record = session.query(MappingRecord).filter(MappingRecord.variant.has(urn=urn)).one_or_none() + assert mapping_record is not None + assert authoritative_allele_for(session, mapping_record) is not None + assert mapping_record.hgvs_assay_level is not None # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 2 for status in annotation_statuses: assert status.annotation_type == "vrs_mapping" - assert status.status == "success" + assert status.disposition == "present" async def test_map_variants_for_score_set_updates_existing_mapped_variants( self, @@ -733,18 +1012,32 @@ async def dummy_mapping_job(): ) session.add(variant) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + score_set_id=variant.score_set_id, + assay_level=SequenceLevel.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) + session.commit() + # Link the prior record to an authoritative allele. Re-mapping must retire this link too, + # not just the record — a live link dangling off a retired record breaks the temporal model. + prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=SequenceLevel.genomic.value) + session.add(prior_allele) + session.commit() + prior_link = MappingRecordAllele( + mapping_record_id=mapping_record.id, + score_set_id=mapping_record.score_set_id, + allele_id=prior_allele.id, + is_authoritative=True, + ) + session.add(prior_link) session.commit() - variant_annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(variant_annotation_status) + session.add(prior_event) session.commit() with ( @@ -766,44 +1059,45 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify the existing mapped variant was marked as non-current - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.id == mapped_variant.id, MappedVariant.current.is_(False)) + # Verify the existing mapping record was retired (valid_to closed) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.id == mapping_record.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None + + # Verify the prior record's authoritative allele link was retired alongside the record, + # so no live link dangles off the retired record. + session.refresh(prior_link) + assert prior_link.valid_to is not None - # Verify a new mapped variant entry was created - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + # Verify a new, live mapping record entry was created + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Gap-free handoff: the prior record closed at exactly the new record's valid_from (one + # supersession timestamp), and the cascade closed the prior link at that same instant — so + # no point-in-time query lands in a hole between the old and new record or their links. + assert non_current_mapping_record.valid_to == new_mapping_record.valid_from + assert prior_link.valid_to == non_current_mapping_record.valid_to - # Verify the non-current annotation status still exists - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter( - VariantAnnotationStatus.variant_id == non_current_mapped_variant.variant_id, - VariantAnnotationStatus.current.is_(False), - ) - .one_or_none() - ) - assert old_annotation_status is not None + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify that a new annotation status was created - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + # Append-only: the prior event is retained and the re-map appends a new one, so the variant + # now has two vrs_mapping events (there is no current flag to flip). + events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(events) == 2 @pytest.mark.integration @@ -862,9 +1156,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -875,19 +1169,19 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -953,9 +1247,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -966,19 +1260,19 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1053,12 +1347,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1130,12 +1424,12 @@ async def dummy_mapping_job(): # Error message originates from our mock mapping construction function assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1206,12 +1500,12 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "Reference metadata missing from mapping results" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1246,20 +1540,21 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( sample_independent_variant_creation_run, ) - # Associate mapped variants with all variants just created in the score set + # Associate mapping records with all variants just created in the score set variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() for variant in variants: - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + score_set_id=variant.score_set_id, + assay_level=SequenceLevel.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(annotation_status) - session.add(mapped_variant) + session.add(prior_event) + session.add(mapping_record) session.commit() async def dummy_mapping_job(): @@ -1295,45 +1590,38 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were marked as non-current and new entries created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == len(variants) * 2 # Each variant has two mapped entries now + # Verify that mapping records were marked as non-current and new entries created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == len(variants) * 2 # Each variant has two mapping records now for variant in variants: - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(False)) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify that annotation statuses where marked as non-current and new entries created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == len(variants) * 2 # Each variant has two annotation statuses now + # Append-only: each variant keeps its prior event and gains a new one from this run. + annotation_events = session.query(AnnotationEvent).all() + assert len(annotation_events) == len(variants) * 2 for variant in variants: - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(False)) - .one_or_none() - ) - assert old_annotation_status is not None - - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + variant_events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(variant_events) == 2 # Verify that the job status was updated. processing_run = ( @@ -1389,12 +1677,12 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1448,12 +1736,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1518,27 +1806,27 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1606,27 +1894,27 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1690,12 +1978,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1744,12 +2032,12 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1775,3 +2063,79 @@ async def dummy_mapping_job(): for job_run in pipeline_run.job_runs: if job_run.id != sample_pipeline_variant_mapping_run.id: assert job_run.status == JobStatus.SKIPPED + + +MAP_MODULE = "mavedb.worker.jobs.variant_processing.mapping" + + +@pytest.mark.unit +def test_map_budget_seconds_floor_scale_and_ceiling(): + # Floor for an empty/tiny set (overhead only), linear scale in between, clamped at the ceiling. + assert _map_budget_seconds(0) == MAP_BUDGET_BASE_SECONDS + assert _map_budget_seconds(48_000) == MAP_BUDGET_BASE_SECONDS + 48_000 + assert _map_budget_seconds(100_000_000) == MAP_BUDGET_MAX_SECONDS + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_keepalive_progress_ticks_and_survives_a_failed_tick(): + """The liveness heartbeat refreshes progress on its cadence and keeps going if a tick fails.""" + ticks = 0 + + def _update(*args, **kwargs): + nonlocal ticks + ticks += 1 + if ticks == 2: # a transient commit failure must not kill the heartbeat + raise RuntimeError("transient progress-commit blip") + + jm = MagicMock() + jm.update_progress.side_effect = _update + jm.logging_context.return_value = {} + + task = asyncio.create_task(_keepalive_progress(jm, 0.001)) + await asyncio.sleep(0.05) + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + assert ticks >= 3 # kept ticking past the failing 2nd tick + assert all(call.args[0] == 30 for call in jm.update_progress.call_args_list) # each tick reasserts 30% liveness + + +@pytest.mark.unit +@pytest.mark.asyncio +async def test_map_variants_for_score_set_budget_timeout_fails_the_job( + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, +): + """A map that outlasts its size-aware budget fails the job, bounding a wedged mapper.""" + + async def slow_map(): + await asyncio.sleep(1) + return {} + + caught = None + result = None + with ( + patch(f"{MAP_MODULE}._map_budget_seconds", return_value=0.05), + patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=slow_map()), + ): + try: + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + except Exception as e: # noqa: BLE001 - asserted on below + caught = e + + assert sample_score_set.mapping_state == MappingState.failed + # Budget-exceeded surfaces as a timeout, whether the decorator re-raises or returns a failed outcome. + if caught is not None: + assert isinstance(caught, TimeoutError) + assert "budget" in str(caught) + else: + assert result.status == JobStatus.FAILED diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py new file mode 100644 index 000000000..52b045788 --- /dev/null +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -0,0 +1,2639 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +import contextlib +from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import timedelta +from unittest.mock import MagicMock, patch + +from variant_annotation.lib.translation.types import ( + ProjectionPair, + TranslationError, + TranslationErrorReason, + TranslationResult, + WtCodonMode, +) + +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.target_category import TargetCategory +from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.projection_group import ProjectionGroup +from mavedb.models.target_accession import TargetAccession +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set +from mavedb.worker.jobs.variant_processing.reverse_translation import ( + _build_translation_config, + cdna_transcript_resolver, + reverse_translate_variants_for_score_set, +) +from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.projection import project_measured_variants +from mavedb.lib.vrs_utils import canonical_variation_document +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.helpers.util.setup.worker import construct_mock_mapping_output + +pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") + +# Module under test, used as the patch root for the external annotation-package calls. +RT_MODULE = "mavedb.worker.jobs.variant_processing.reverse_translation" + + +class FakeVrsVariation: + """Stand-in for a ga4gh.vrs variation returned by translate_hgvs_to_variation. + + The reverse translation job only reads ``.id`` (for the VRS digest / dedup key) + and ``.model_dump()`` (persisted into the ``post_mapped`` JSONB column), so the + fake exposes exactly those. ``vrs_type`` lets a test assert whether a candidate + became a single Allele or a CisPhasedBlock. + """ + + def __init__(self, vrs_id: str, vrs_type: str = "Allele"): + self.id = vrs_id + self.vrs_type = vrs_type + + def model_dump(self, **kwargs) -> dict: + # Accept (and ignore) model_dump kwargs like exclude_none, mirroring the real VRS model. + return {"type": self.vrs_type, "id": self.id} + + +def fake_construct( + results_by_hgvs: dict, + errors_by_hgvs: dict | None = None, + not_translatable_by_hgvs: dict | None = None, + upstream_unavailable_by_hgvs: dict | None = None, +): + """Build a stand-in for ``construct_equivalent_variants``. + + The job correlates each TranslationResult/TranslationError back to its originating + mapping record via object identity on ``result.input``, so the fake must echo the + exact VariantInput instances it was handed. Results are keyed by the input's + assay-level HGVS string. + + Each result spec is one of: + + * a list of ``(hgvs_c, hgvs_g)`` pairs — one ProjectionPair each. + ``hgvs_g`` may be ``None`` for a projection-failed, coding-only projection pair. + * a ``(pairs, hgvs_p)`` tuple pairing that list with a shared protein apex. + + Each pair becomes one ProjectionPair, mirroring the position-aligned + coding↔genomic rows the real reverse-translate CLI emits (with ``--one-row-per-input``). + + ``errors_by_hgvs`` maps an assay HGVS to a genuine-failure message (reason ``FAILED``); + ``not_translatable_by_hgvs`` maps one to a benign non-translatable message (reason + ``NOT_TRANSLATABLE``), standing in for an edit type the library screens out up front; + ``upstream_unavailable_by_hgvs`` maps one to a UTA outage message (reason ``UPSTREAM_UNAVAILABLE``). + """ + errors_by_hgvs = errors_by_hgvs or {} + not_translatable_by_hgvs = not_translatable_by_hgvs or {} + upstream_unavailable_by_hgvs = upstream_unavailable_by_hgvs or {} + + def _construct(inputs, *, transcripts, coordinates, config): + results, errors = [], [] + for inp in inputs: + if inp.hgvs in not_translatable_by_hgvs: + errors.append( + TranslationError( + input=inp, + error=not_translatable_by_hgvs[inp.hgvs], + reason=TranslationErrorReason.NOT_TRANSLATABLE, + ) + ) + elif inp.hgvs in upstream_unavailable_by_hgvs: + errors.append( + TranslationError( + input=inp, + error=upstream_unavailable_by_hgvs[inp.hgvs], + reason=TranslationErrorReason.UPSTREAM_UNAVAILABLE, + ) + ) + elif inp.hgvs in errors_by_hgvs: + errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) + elif inp.hgvs in results_by_hgvs: + spec = results_by_hgvs[inp.hgvs] + # A bare list is the candidate pairs; a tuple pairs them with a protein apex. + pairs, hgvs_p = (spec[0], spec[1]) if isinstance(spec, tuple) else (spec, None) + results.append( + TranslationResult( + input=inp, + projection_pairs=[ProjectionPair(hgvs_c=hgvs_c, hgvs_g=hgvs_g) for hgvs_c, hgvs_g in pairs], + hgvs_p=hgvs_p, + ) + ) + else: + errors.append(TranslationError(input=inp, error=f"no stub result for {inp.hgvs!r}")) + return results, errors + + return _construct + + +def fake_translate(id_by_hgvs: dict, type_by_hgvs: dict | None = None, errors_by_hgvs: dict | None = None): + """Build a stand-in for ``translate_hgvs_to_variation`` mapping candidate HGVS -> VRS id. + + Unknown HGVS get a deterministic id derived from the string so distinct candidates + stay distinct; supplying explicit ids lets a test force two candidates to collapse. + ``type_by_hgvs`` lets a test mark a bracketed candidate as a ``CisPhasedBlock`` (the + real translate_hgvs_to_variation returns one for cis-phased multivariants). + ``errors_by_hgvs`` makes a candidate raise, standing in for an HGVS form ga4gh cannot + translate to VRS. + """ + type_by_hgvs = type_by_hgvs or {} + errors_by_hgvs = errors_by_hgvs or {} + + def _translate(hgvs: str, translator=None) -> FakeVrsVariation: + if hgvs in errors_by_hgvs: + raise ValueError(errors_by_hgvs[hgvs]) + return FakeVrsVariation( + id_by_hgvs.get(hgvs, f"ga4gh:VA.{hgvs}"), + type_by_hgvs.get(hgvs, "Allele"), + ) + + return _translate + + +class FakeCoordinates: + """Deterministic stand-in for ``WorkerCoordinateTranslator``, used by the measured projection step. + + The worker context's ``hdp`` is a bare mock, so the real translator would produce projection errors + rather than strings. Tables pin a projection per input HGVS; anything else projects to a fixed + placeholder so a test that doesn't care still gets a well-formed measured group. + """ + + def __init__( + self, + c_by_g: dict | None = None, + g_by_c: dict | None = None, + p_by_c: dict | None = None, + mane: str | None = None, + ): + self.c_by_g = c_by_g or {} + self.g_by_c = g_by_c or {} + self.p_by_c = p_by_c or {} + self.mane = mane + self.mane_lookups: list[str] = [] + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + return self.c_by_g.get(g_hgvs, f"{transcript}:c.1A>G") + + def normalize(self, hgvs_string: str) -> str: + return hgvs_string + + def c_to_g(self, c_hgvs: str) -> str: + return self.g_by_c.get(c_hgvs, "NC_000001.11:g.1000A>G") + + def c_to_p(self, c_hgvs: str) -> str: + return self.p_by_c.get(c_hgvs, "NP_999999.1:p.(Met1Val)") + + # The tables hold changes whose spans match on both levels, so the unnormalized projections agree. + def c_to_g_literal(self, c_hgvs: str) -> str: + return self.c_to_g(c_hgvs) + + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + return self.g_to_c(g_hgvs, transcript) + + def c_to_n(self, c_hgvs: str) -> str: + return c_hgvs.replace(":c.", ":n.") + + def mane_select_transcript(self, g_hgvs: str) -> str | None: + self.mane_lookups.append(g_hgvs) + return self.mane + + +# Mapping recomputes an incoming allele's identity from its own content rather than trusting the +# producer's `id`, so a stored authoritative allele carries this rather than the fixture's placeholder. +# The fold-in tests below depend on the reverse-translated coding allele identifying to the same value. +CANONICAL_AUTHORITATIVE_DIGEST = canonical_variation_document( + TEST_VALID_POST_MAPPED_VRS_ALLELE, subject="mapping fixture" +)[1] + + +# Candidates for the rerun tests: the measured coding allele with its genomic partner, and an unrelated pair. +MEASURED_C = "NM_000001.1:c.5A>G" +MEASURED_PARTNER_G = "NC_000001.11:g.1000A>G" +OTHER_C = "NM_000001.1:c.9C>T" +OTHER_G = "NC_000001.11:g.1004C>T" + + +async def _map_variants(session, mock_worker_ctx, mapping_run, score_set, with_layers=frozenset({"g", "c", "p"})): + """Run the (mocked) mapping job so the score set's variants gain authoritative + MappingRecords/Alleles — the real inputs the reverse translation job reads.""" + + async def dummy_mapping_job(): + return await construct_mock_mapping_output(session=session, score_set=score_set, with_layers=with_layers) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], mapping_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + session.commit() + + +async def _reverse_translate(session, mock_worker_ctx, rt_run, coordinates=None): + # The job opens a UTA-backed TranscriptSource to back codon_at (WtCodonMode.ALL). + # construct_equivalent_variants is mocked in these tests, so the source is never + # queried -- stub the factory with a no-op context yielding a dummy client. The measured + # projection step reads the coordinate translator, so it gets a deterministic fake. + coordinates = coordinates or FakeCoordinates() + with ( + patch(f"{RT_MODULE}.uta_transcript_source", lambda: contextlib.nullcontext(MagicMock())), + patch(f"{RT_MODULE}.WorkerCoordinateTranslator", lambda hdp: coordinates), + ): + return await reverse_translate_variants_for_score_set( + mock_worker_ctx, + rt_run.id, + JobManager(session, mock_worker_ctx["redis"], rt_run.id), + ) + + +def _cross_level_events(session, score_set_id, disposition=None, reason=None): + query = ( + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter( + Variant.score_set_id == score_set_id, + AnnotationEvent.annotation_type == "cross_level_translation", + ) + ) + if disposition is not None: + query = query.filter(AnnotationEvent.disposition == disposition) + if reason is not None: + query = query.filter(AnnotationEvent.reason == reason) + return query.all() + + +def _non_authoritative_links(session): + return session.query(MappingRecordAllele).filter(MappingRecordAllele.is_authoritative.is_(False)).all() + + +def _live_links(session, record_id): + """Every live (valid_to IS NULL) link for a mapping record, authoritative or derived.""" + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.valid_to.is_(None), + ) + .all() + ) + + +def _authoritative_link(session, record_id): + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .one() + ) + + +def _record_for(session, variant_id): + return session.query(MappingRecord).filter(MappingRecord.variant_id == variant_id).one() + + +def _group(session, group_id): + return session.get(ProjectionGroup, group_id) + + +def _measured_group(session, record_id): + """The projection group the record's live authoritative link points at.""" + return _group(session, _authoritative_link(session, record_id).projection_group) + + +def _live_groups(session, record_id): + return ( + session.query(ProjectionGroup) + .filter(ProjectionGroup.mapping_record_id == record_id, ProjectionGroup.valid_to.is_(None)) + .all() + ) + + +def _authoritative_versions(session, record_id): + """Every version of the record's authoritative link, oldest first.""" + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.is_authoritative.is_(True), + ) + .order_by(MappingRecordAllele.id) + .all() + ) + + +@pytest.mark.unit +class TestBuildTranslationConfig: + """Unit tests for _build_translation_config — the optional translation_config job param.""" + + def test_none_uses_job_defaults(self): + config = _build_translation_config(None) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_defaults_ride_out_a_minutes_long_uta_outage(self): + config = _build_translation_config(None) + assert config.upstream_max_attempts == 6 + assert config.upstream_retry_backoff_seconds == 5.0 + + def test_empty_dict_uses_job_defaults(self): + config = _build_translation_config({}) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_overrides_win_and_string_wt_codon_mode_is_coerced(self): + config = _build_translation_config({"wt_codon_mode": "unambiguous", "max_indel_size": 7}) + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + assert config.include_indels is True # untouched default preserved + + def test_can_disable_indels_with_none_mode(self): + config = _build_translation_config({"wt_codon_mode": "none", "include_indels": False}) + assert config.wt_codon_mode == WtCodonMode.NONE + assert config.include_indels is False + + def test_rejects_invalid_wt_codon_mode_with_actionable_message(self): + with pytest.raises(ValueError, match=r"wt_codon_mode 'bogus'.*Valid values"): + _build_translation_config({"wt_codon_mode": "bogus"}) + + def test_rejects_wt_codon_mode_without_indels(self): + # TranslationConfig enforces: a wt_codon_mode other than "none" requires include_indels. + with pytest.raises(ValueError, match="translation_config"): + _build_translation_config({"wt_codon_mode": "all", "include_indels": False}) + + def test_rejects_unknown_field_and_lists_valid_options(self): + with pytest.raises( + ValueError, match=r"Unknown translation_config option\(s\): not_a_real_field.*Valid options" + ): + _build_translation_config({"not_a_real_field": 1}) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestReverseTranslateVariantsForScoreSetUnit: + """Unit tests for the reverse_translate_variants_for_score_set job.""" + + async def test_no_mapping_records_is_a_noop( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no current, post-mapped mapping records there is nothing to translate.""" + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0, "fold_in_missed": 0} + + # No candidate alleles, links, or annotations were produced. + assert session.query(Allele).count() == 0 + assert _non_authoritative_links(session) == [] + assert _cross_level_events(session, sample_score_set.id) == [] + + async def test_same_day_remap_without_a_cdna_row_skips_rather_than_binding_a_stale_transcript( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + map_variants_sample_params, + ): + """#763: a record binds its transcript to its own run via job_run_id. When a later same-day remap + emits no cdna row, reverse translation must resolve no transcript and skip -- not fall back to an + earlier same-day run's cdna transcript, which the day-granular key alone could not tell apart.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + + # Run 1 (all layers) writes a cdna TargetGeneMapping; its transcript must not leak to run 2. + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Run 2 -- same score set, same calendar day -- emits genomic + protein only, no cdna row. Its + # records supersede run 1's and are the current ones reverse translation reads. + second_mapping_run = JobRun( + urn="test:map_variants_for_score_set:2", + job_type="map_variants_for_score_set", + job_function="map_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=map_variants_sample_params, + ) + session.add(second_mapping_run) + session.commit() + await _map_variants(session, mock_worker_ctx, second_mapping_run, sample_score_set, with_layers={"g", "p"}) + + # construct_equivalent_variants is never reached for a skipped variant. + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["translated"] == 0 + assert result.data["skipped"] == 1 + assert result.data["alleles_created"] == 0 + assert _non_authoritative_links(session) == [] + assert len(_cross_level_events(session, sample_score_set.id, reason="transcript_unresolved")) == 1 + + async def test_cis_phased_assay_hgvs_is_skipped_before_the_engine( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A cis-phased multivariant assay HGVS (``g.[a;b]``) cannot be forward-translated -- the + engine's parser rejects the allele list at the opening ``[``. The job screens it out up front + and records a benign NOT_APPLICABLE / cis_phased_unsupported skip, never handing it to the + engine (which would otherwise count it as a failure the drop gate reads as a regression).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Map first so the variant gains a transcript-bearing authoritative record: the skip must fire + # on the cis-phased shape even though a transcript resolves, not as a transcript-unresolved skip. + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + cis_phased = "NC_000016.10:g.[1314031A>T;1320511T>C]" + record = _record_for(session, variant.id) + record.hgvs_assay_level = cis_phased + session.commit() + + def _fail_if_engine_sees_it(inputs, *, transcripts, coordinates, config): + assert list(inputs) == [], "a cis-phased input must be screened out before the engine" + return [], [] + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", _fail_if_engine_sees_it), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} + assert _non_authoritative_links(session) == [] + + events = _cross_level_events(session, sample_score_set.id, reason="cis_phased_unsupported") + assert len(events) == 1 + assert events[0].disposition == "not_applicable" + assert events[0].event_metadata["hgvs_input"] == cis_phased + + async def test_falls_back_to_mapped_date_when_the_record_has_no_job_run_id( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A record with no job_run_id (pre-column, or reshaped from legacy) resolves its cdna transcript + via the day-granular (target_gene_id, mapped_date) fallback rather than skipping.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Simulate the pre-job_run_id / reshaped state: no TargetGeneMapping carries a run anchor, so + # resolution has only the day-granular key to work with. + session.query(TargetGeneMapping).filter( + TargetGeneMapping.target_gene_id == sample_score_set.target_genes[0].id + ).update({TargetGeneMapping.job_run_id: None}, synchronize_session=False) + session.commit() + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # The transcript resolved via the fallback, so the variant translated instead of skipping. + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} + + @pytest.mark.parametrize("record_has_job_run_id", [False, True]) + async def test_legacy_record_falls_back_to_the_target_post_mapped_cdna_accession( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + record_has_job_run_id, + ): + """Legacy mappings wrote no cdna TargetGeneMapping; their coding accession lives in the target's + post_mapped_metadata. A record with no job_run_id falls back to it, but a record anchored to a run + resolves only through that run's own cdna row, so the legacy accession never leaks into it.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic only: no cdna TargetGeneMapping for either key to find. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + target_gene.post_mapped_metadata = {"cdna": {"sequence_accessions": ["NM_777777.1", "NM_777778.1"]}} + if not record_has_job_run_id: + session.query(TargetGeneMapping).filter(TargetGeneMapping.target_gene_id == target_gene.id).update( + {TargetGeneMapping.job_run_id: None}, synchronize_session=False + ) + session.commit() + + record = _record_for(session, variant.id) + tgm = session.get(TargetGeneMapping, record.target_gene_mapping_id) + resolve = cdna_transcript_resolver(session, sample_score_set.id, FakeCoordinates()) + expected = None if record_has_job_run_id else "NM_777777.1" + assert resolve(tgm.job_run_id, target_gene.id, tgm.mapped_date, "NC_000001.11:g.1000A>G") == expected + + assay_hgvs = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: [("NM_777777.1:c.5A>G", None)]}) + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["transcripts"] = [inp.transcript for inp in inputs] + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + if record_has_job_run_id: + assert captured["transcripts"] == [] + assert result.data["skipped"] == 1 + assert len(_cross_level_events(session, sample_score_set.id, reason="transcript_unresolved")) == 1 + else: + assert captured["transcripts"] == ["NM_777777.1"] + assert result.data["translated"] == 1 + + @pytest.mark.parametrize("record_has_job_run_id", [True, False]) + async def test_genomic_only_target_falls_back_to_its_transcript_accession_then_mane_select( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + record_has_job_run_id, + ): + """A genomic-only target has no cdna row and no legacy cdna accession. A transcript target + accession is used as is; otherwise a protein-coding target takes the MANE Select transcript at the + record's locus, looked up once per target gene.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + target_gene.category = TargetCategory.protein_coding + target_gene.post_mapped_metadata = {"genomic": {"sequence_accessions": ["NC_000001.11"]}} + if not record_has_job_run_id: + session.query(TargetGeneMapping).filter(TargetGeneMapping.target_gene_id == target_gene.id).update( + {TargetGeneMapping.job_run_id: None}, synchronize_session=False + ) + session.commit() + record = _record_for(session, variant.id) + tgm = session.get(TargetGeneMapping, record.target_gene_mapping_id) + key = (tgm.job_run_id, target_gene.id, tgm.mapped_date) + + coordinates = FakeCoordinates(mane="NM_888888.2") + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + assert resolve(*key, "NC_000001.11:g.1000A>G") == "NM_888888.2" + assert resolve(*key, "NC_000001.11:g.1001A>G") == "NM_888888.2" + assert coordinates.mane_lookups == ["NC_000001.11:g.1000A>G"] + + # Protein records pass no genomic HGVS and keep their NP_→NM_ lookup. + assert resolve(*key, None) is None + + target_gene.target_accession = TargetAccession(accession="NM_999999.3", assembly="GRCh38", gene="GENE") + session.commit() + coordinates = FakeCoordinates(mane="NM_888888.2") + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + assert resolve(*key, "NC_000001.11:g.1000A>G") == "NM_999999.3" + assert coordinates.mane_lookups == [] + + async def test_mane_select_is_only_looked_up_for_protein_coding_targets_and_a_bounded_number_of_times( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + target_gene.post_mapped_metadata = {"genomic": {"sequence_accessions": ["NC_000001.11"]}} + target_gene.category = TargetCategory.regulatory + session.commit() + record = _record_for(session, variant.id) + tgm = session.get(TargetGeneMapping, record.target_gene_mapping_id) + key = (tgm.job_run_id, target_gene.id, tgm.mapped_date) + + coordinates = FakeCoordinates(mane="NM_888888.2") + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + assert resolve(*key, "NC_000001.11:g.1000A>G") is None + assert coordinates.mane_lookups == [] + + target_gene.category = TargetCategory.protein_coding + session.commit() + coordinates = FakeCoordinates(mane=None) + resolve = cdna_transcript_resolver(session, sample_score_set.id, coordinates) + for position in range(1000, 1010): + assert resolve(*key, f"NC_000001.11:g.{position}A>G") is None + assert len(coordinates.mane_lookups) == 5 + + @pytest.mark.parametrize("failure", ["projection", "transcript"]) + async def test_a_failed_projection_keeps_the_group_an_earlier_run_wrote( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + failure, + ): + """A data provider outage must not erase coding and protein projections: the record keeps its group.""" + assay_hgvs = "NC_000001.11:g.1000A>G" + variant = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt=assay_hgvs, data={} + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + record = _record_for(session, variant.id) + + def resolve(*_): + return "NM_000001.1" + + project_measured_variants( + session, + sample_score_set.id, + coordinates=FakeCoordinates(c_by_g={assay_hgvs: MEASURED_C}), + resolve_transcript=resolve, + ) + session.commit() + written = _measured_group(session, record.id) + assert written.hgvs_c == MEASURED_C + + class Unreachable(FakeCoordinates): + def g_to_c(self, g_hgvs, transcript): + raise ConnectionError("cdot-rest unreachable") + + def unreachable_resolve(*_): + raise ConnectionError("cdot-rest unreachable") + + counts = project_measured_variants( + session, + sample_score_set.id, + coordinates=Unreachable() if failure == "projection" else FakeCoordinates(), + resolve_transcript=resolve if failure == "projection" else unreachable_resolve, + ) + session.commit() + + assert counts["kept_on_error"] == 1 + assert _measured_group(session, record.id).id == written.id + assert [group.id for group in _live_groups(session, record.id)] == [written.id] + + async def test_untranslatable_delins_still_gets_a_measured_projection_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A multi-codon delins has no codon-level equivalence class, so the library returns NOT_TRANSLATABLE + and RT links nothing. The measured group comes from the projection step, not from RT, so the record + still describes its variant at the coding and protein levels.""" + assay_hgvs = "NC_000001.11:g.1000_1005delinsTTT" + variant = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt=assay_hgvs, data={} + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g", "c"} + ) + + hgvs_c = "NM_999999.1:c.4_9delinsTTT" + coordinates = FakeCoordinates( + c_by_g={assay_hgvs: hgvs_c}, p_by_c={hgvs_c: "NP_999999.1:p.(Lys2_Ala3delinsPhe)"} + ) + construct = fake_construct({}, not_translatable_by_hgvs={assay_hgvs: "multi-codon delins"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate( + session, mock_worker_ctx, sample_independent_reverse_translation_run, coordinates=coordinates + ) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["skipped"] == 1 + assert _non_authoritative_links(session) == [] + + record = _record_for(session, variant.id) + measured = _measured_group(session, record.id) + assert measured.current + assert (measured.hgvs_c, measured.hgvs_g, measured.hgvs_p) == ( + hgvs_c, + assay_hgvs, + "NP_999999.1:p.Lys2_Ala3delinsPhe", + ) + assert [group.id for group in _live_groups(session, record.id)] == [measured.id] + + async def test_creates_genomic_and_coding_candidate_alleles( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A mapped variant expands into one non-authoritative allele per equivalence-class + candidate, each linked to the originating mapping record. The two members of a single + projection pair (coding + genomic projection) share one projection_group.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} + + # Two non-authoritative links were created, both for our one mapping record. + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 2 + + genomic_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.genomic").one() + assert genomic_allele.level == SequenceLevel.genomic.value + assert genomic_allele.hgvs_g == g_candidate + assert genomic_allele.hgvs_c is None + assert genomic_allele.transcript == "NC_000001.11" + assert genomic_allele.post_mapped == {"type": "Allele", "id": "ga4gh:VA.genomic"} + + coding_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.coding").one() + assert coding_allele.level == SequenceLevel.cdna.value + assert coding_allele.hgvs_c == c_candidate + assert coding_allele.hgvs_g is None + assert coding_allele.transcript == "NM_000001.1" + + # The candidate alleles are linked to the same mapping record as the authoritative allele. + mapping_record = _record_for(session, variant.id) + assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} + + # Both members of the single projection pair share one candidate group, distinct from the measured + # group, whose strings are the pair's own. + (group_id,) = {link.projection_group for link in non_auth_links} + assert group_id != _authoritative_link(session, mapping_record.id).projection_group + group = _group(session, group_id) + assert group.current + assert (group.hgvs_c, group.hgvs_g, group.hgvs_p) == (c_candidate, g_candidate, None) + + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 1 + assert events[0].disposition == "present" + # The event metadata carries the paired projection pair shape, not the removed flat lists. + assert events[0].event_metadata["candidates"] == [ + {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None, "measured": False} + ] + + async def test_transcript_is_queryable_via_derived_expression( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Allele.transcript is a derived hybrid_property — filtering on it in SQL resolves to + the accession of the populated HGVS column, with no stored transcript column.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # The SQL expression derives the accession from the populated HGVS column. + genomic = session.query(Allele).filter(Allele.transcript == "NC_000001.11").all() + assert [a.vrs_digest for a in genomic] == ["ga4gh:VA.genomic"] + coding = session.query(Allele).filter(Allele.transcript == "NM_000001.1").all() + assert [a.vrs_digest for a in coding] == ["ga4gh:VA.coding"] + + async def test_protein_consequence_is_persisted_as_a_protein_allele( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The deterministic protein consequence (``result.hgvs_p``) is emitted as the + protein-level member of the equivalence set -- a non-authoritative ``level=protein`` + allele linked to the record, not dropped. It is the measured variant's own consequence, so its + link joins the record's measured group; each candidate group records it as its hgvs_p.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + # c_to_p emits a predicted consequence in parens; the job strips them before use. + p_consequence = "NP_000001.1:p.(Met1Val)" + p_stripped = "NP_000001.1:p.Met1Val" + construct = fake_construct({assay_hgvs: ([(c_candidate, g_candidate)], p_consequence)}) + translate = fake_translate( + { + g_candidate: "ga4gh:VA.genomic", + c_candidate: "ga4gh:VA.coding", + p_stripped: "ga4gh:VA.protein", # the stripped form is what reaches the translator + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() + assert protein_allele.level == SequenceLevel.protein.value + assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens + + # Linked to the record as a non-authoritative member of the measured group. + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + protein_link = next(link for link in _non_authoritative_links(session) if link.allele_id == protein_allele.id) + assert protein_link.projection_group == measured_group_id + # The coding/genomic pair shares one candidate group, which carries the stripped consequence. + pair_links = [link for link in _non_authoritative_links(session) if link.allele_id != protein_allele.id] + (pair_group_id,) = {link.projection_group for link in pair_links} + assert pair_group_id != measured_group_id + pair_group = _group(session, pair_group_id) + assert (pair_group.hgvs_c, pair_group.hgvs_g, pair_group.hgvs_p) == (c_candidate, g_candidate, p_stripped) + + async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A bracketed genomic projection (non-adjacent codon components) is translated into a + CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest, and + grouped with its coding partner in the projection pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + block_candidate = "NC_000001.11:g.[1000A>G;1002T>C]" + construct = fake_construct({assay_hgvs: [(c_candidate, block_candidate)]}) + translate = fake_translate( + {c_candidate: "ga4gh:VA.coding", block_candidate: "ga4gh:CPB.block"}, + type_by_hgvs={block_candidate: "CisPhasedBlock"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} + + block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() + assert block_allele.level == SequenceLevel.genomic.value + # The full bracketed expression is stored verbatim; its accession anchors the row. + assert block_allele.hgvs_g == block_candidate + assert block_allele.transcript == "NC_000001.11" + assert block_allele.post_mapped == {"type": "CisPhasedBlock", "id": "ga4gh:CPB.block"} + + # The block (genomic projection) and its coding partner are one projection pair -> one group. + (group_id,) = {link.projection_group for link in _non_authoritative_links(session)} + group = _group(session, group_id) + assert (group.hgvs_c, group.hgvs_g) == (c_candidate, block_candidate) + + async def test_duplicate_candidate_digests_are_deduped_per_record( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When the coding and genomic members of a projection pair resolve to the same VRS digest, + only one allele/link is written for that mapping record (per the library's dedup contract). + The surviving link still carries the pair's group.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + # Both members of the projection pair collapse to the same VRS digest. + translate = fake_translate({g_candidate: "ga4gh:VA.shared", c_candidate: "ga4gh:VA.shared"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 1 + group = _group(session, non_auth_links[0].projection_group) + assert (group.hgvs_c, group.hgvs_g) == (c_candidate, g_candidate) + assert session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.shared").count() == 1 + + async def test_authoritative_allele_is_folded_into_its_projection_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Nucleotide assay: the measured allele appears among the RT candidates (the library makes + no exclusions). That pair is the measured variant itself, so rather than forming a candidate + group its other member joins the record's measured group, which then resolves to {canonical + coding, canonical genomic}. The authoritative link already points at that group and is not + re-versioned by RT.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Exactly one allele exists after mapping: the authoritative (measured) allele. + assert session.query(Allele).count() == 1 + assert session.query(Allele).one().vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_sibling = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_sibling)]}) + # The coding member IS the measured allele (folds in); the genomic member is its projection. + translate = fake_translate({c_candidate: CANONICAL_AUTHORITATIVE_DIGEST, g_sibling: "ga4gh:VA.genomic"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + # Only the genomic sibling is a newly created derived allele; the coding member folds into + # the existing authoritative link rather than creating a second link/allele. + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 0} + + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + assert len(derived_links) == 1 + + # The sibling derived link joins the measured group -- the pairing is resolvable from the + # authoritative allele's group -- and no candidate group was created for the measured pair. + measured_group_id = auth_link.projection_group + assert measured_group_id is not None + assert derived_links[0].projection_group == measured_group_id + assert [group.id for group in _live_groups(session, record.id)] == [measured_group_id] + + # The measured group resolves to {canonical c (authoritative), canonical g (projection)}: two live + # links, one authoritative and one derived. + group_links = [link for link in _live_links(session, record.id) if link.projection_group == measured_group_id] + assert len(group_links) == 2 + assert {link.is_authoritative for link in group_links} == {True, False} + assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.genomic" + + # Only the mapping job's link and the projection step's copy exist: RT did not regroup it. + assert [link.projection_group for link in _authoritative_versions(session, record.id)] == [ + None, + measured_group_id, + ] + + metadata = _cross_level_events(session, sample_score_set.id)[0].event_metadata + assert metadata["fold_in_missed"] is False + assert metadata["candidates"] == [ + {"hgvs_c": c_candidate, "hgvs_g": g_sibling, "variant_type": None, "measured": True} + ] + + async def test_authoritative_fold_in_when_measured_at_genomic_level( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The fold-in is symmetric in the measured level: when the genomic member of a projection pair + equals the record's authoritative allele, the coding sibling joins the measured group and no + candidate group is created for the pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity mapping so RT resolves a coding transcript. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g", "c"} + ) + assert session.query(Allele).one().vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_sibling = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_sibling, g_candidate)]}) + # The genomic member IS the measured allele (folds in); the coding member is its projection. + translate = fake_translate({c_sibling: "ga4gh:VA.coding", g_candidate: CANONICAL_AUTHORITATIVE_DIGEST}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 0} + + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + measured_group_id = auth_link.projection_group + assert measured_group_id is not None + assert [link.projection_group for link in derived_links] == [measured_group_id] + assert [group.id for group in _live_groups(session, record.id)] == [measured_group_id] + assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.coding" + + async def test_measured_allele_matching_no_candidate_is_flagged_as_a_missed_fold_in( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Nucleotide assay whose measured allele has a different digest from every RT candidate: the + measured group holds only the authoritative link, both candidates form a candidate group, and the run + counts the miss and flags it on the variant's event.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_sibling = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_sibling)]}) + # The coding member is the measured change, but it identifies differently from the stored allele. + translate = fake_translate({c_candidate: "ga4gh:VA.divergent", g_sibling: "ga4gh:VA.genomic"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + assert [ + link.is_authoritative + for link in _live_links(session, record.id) + if link.projection_group == measured_group_id + ] == [True] + (candidate_group_id,) = {link.projection_group for link in _non_authoritative_links(session)} + assert candidate_group_id != measured_group_id + assert _cross_level_events(session, sample_score_set.id)[0].event_metadata["fold_in_missed"] is True + + async def test_projection_failed_candidate_is_a_one_member_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A projection pair whose genomic projection failed (hgvs_g is None) yields a well-formed + one-member (coding-only) group -- no crash, no desync, and the group holds just the coding + link. This is the case the old two-list shape silently dropped.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_paired = "NM_000001.1:c.5A>G" + g_paired = "NC_000001.11:g.1000A>G" + c_orphan = "NM_000001.1:c.9A>T" # projection failed: no genomic partner + construct = fake_construct({assay_hgvs: [(c_paired, g_paired), (c_orphan, None)]}) + translate = fake_translate( + { + c_paired: "ga4gh:VA.coding0", + g_paired: "ga4gh:VA.genomic0", + c_orphan: "ga4gh:VA.coding1", + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 3, "fold_in_missed": 1} + + non_auth_links = _non_authoritative_links(session) + groups: dict[tuple[str, str | None], set[str]] = {} + for link in non_auth_links: + group = _group(session, link.projection_group) + groups.setdefault((group.hgvs_c, group.hgvs_g), set()).add(link.allele.vrs_digest) + + # One group is the full projection pair; the other is the projection-failed coding-only member. + assert groups == { + (c_paired, g_paired): {"ga4gh:VA.coding0", "ga4gh:VA.genomic0"}, + (c_orphan, None): {"ga4gh:VA.coding1"}, + } + + async def test_protein_assay_measured_group_holds_only_the_protein_over_a_fan_out( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Protein assay: the authoritative allele is the protein itself, which appears in no + coding/genomic projection pair, so nothing folds in and its measured group describes only + the protein. The ambiguous fan-out becomes N distinct candidate groups, each a coding/genomic pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: the authoritative allele is the protein allele. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + assert session.query(Allele).one().vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST + + assay_hgvs = "NP_000000.1:p.Met1Val" + # A degenerate protein consequence -> three coding/genomic projection pairs (the fan-out). + pairs = [ + ("NM_000001.1:c.3A>G", "NC_000001.11:g.100A>G"), + ("NM_000001.1:c.3A>C", "NC_000001.11:g.100A>C"), + ("NM_000001.1:c.3A>T", "NC_000001.11:g.100A>T"), + ] + # Protein assay: no protein apex is re-emitted (result.hgvs_p is None) -- the protein + # allele is already authoritative. + construct = fake_construct({assay_hgvs: pairs}) + translate = fake_translate({hgvs: f"ga4gh:VA.{hgvs}" for pair in pairs for hgvs in pair}) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000001.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 6, "fold_in_missed": 0} + + record = _record_for(session, variant.id) + # The protein authoritative allele is not part of any pair: its measured group holds only hgvs_p and + # no derived link joins it. + auth_link = _authoritative_link(session, record.id) + assert auth_link.allele.vrs_digest == CANONICAL_AUTHORITATIVE_DIGEST + measured = _group(session, auth_link.projection_group) + assert (measured.hgvs_c, measured.hgvs_g, measured.hgvs_p) == (None, None, assay_hgvs) + + # Three distinct candidate groups, each a coding/genomic pair (two members), with no protein + # consequence re-emitted. + groups: dict[tuple[str, str, str | None], int] = {} + for link in _non_authoritative_links(session): + assert link.projection_group != measured.id + group = _group(session, link.projection_group) + key = (group.hgvs_c, group.hgvs_g, group.hgvs_p) + groups[key] = groups.get(key, 0) + 1 + assert groups == {(hgvs_c, hgvs_g, None): 2 for hgvs_c, hgvs_g in pairs} + + async def test_independent_rerun_supersedes_prior_links_and_candidate_groups( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """An independent second run is idempotent: it retires the prior derived links and candidate + groups (closing valid_to) and writes fresh live ones, so the set of *current* non-authoritative + links is stable while the superseded rows are retained as history -- the partial unique index on + live links is never violated and no alleles are duplicated. The measured group, whose strings are + unchanged, is left alone.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + # First run: two live derived links sharing one candidate group. + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert first.status == JobStatus.SUCCEEDED + first_links = _non_authoritative_links(session) + assert len(first_links) == 2 + assert all(link.valid_to is None for link in first_links) + (first_group_id,) = {link.projection_group for link in first_links} + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + + # Second, independent run with identical inputs. + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + second = await _reverse_translate(session, mock_worker_ctx, rerun) + + assert second.status == JobStatus.SUCCEEDED + + links = _non_authoritative_links(session) + live = [link for link in links if link.valid_to is None] + retired = [link for link in links if link.valid_to is not None] + + # The live set is stable (still exactly the two candidates, sharing a fresh candidate group) and + # the prior run's links and group are retained as closed history rather than deleted. + assert len(live) == 2 + assert len(retired) == 2 + (second_group_id,) = {link.projection_group for link in live} + assert second_group_id != first_group_id + first_group, second_group = _group(session, first_group_id), _group(session, second_group_id) + assert first_group.valid_to == second_group.valid_from + assert (second_group.hgvs_c, second_group.hgvs_g) == (first_group.hgvs_c, first_group.hgvs_g) + + # The measured group and the authoritative link pointing at it are untouched by the rerun. + assert _authoritative_link(session, record.id).projection_group == measured_group_id + assert _group(session, measured_group_id).current + assert len(_authoritative_versions(session, record.id)) == 2 + + # Gap-free handoff: the prior links closed at exactly the instant the new links opened, under + # one supersession timestamp -- so a point-in-time query never lands in a hole between runs, + # even though the translation loop ran (and could have committed) between retire and insert. + assert {link.valid_to for link in retired} == {link.valid_from for link in live} + assert len({link.valid_from for link in live}) == 1 + + # The two candidate alleles are reused across runs, not duplicated. + assert session.query(Allele).filter(Allele.vrs_digest.in_(["ga4gh:VA.genomic", "ga4gh:VA.coding"])).count() == 2 + + # The cross-level translation log is append-only: the rerun adds a second event, and + # "current" is the latest by id (there is no current flag to desync). + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 2 + assert max(events, key=lambda e: e.id).disposition == "present" + + @pytest.mark.parametrize( + "rerun_pairs, rerun_unavailable, expected_live, candidate_group_kept", + [ + # The measured pair moves position: its partner still joins the measured group. + ( + [(OTHER_C, OTHER_G), (MEASURED_C, MEASURED_PARTNER_G)], + False, + { + ("ga4gh:VA.partner", "measured"), + ("ga4gh:VA.other_c", "candidate"), + ("ga4gh:VA.other_g", "candidate"), + }, + False, + ), + # The measured allele no longer appears among the candidates: its partner leaves the measured group. + ( + [(OTHER_C, OTHER_G)], + False, + {("ga4gh:VA.other_c", "candidate"), ("ga4gh:VA.other_g", "candidate")}, + False, + ), + # Identical candidates: the candidate group is still re-minted (TODO#765), the measured group is not. + ( + [(MEASURED_C, MEASURED_PARTNER_G), (OTHER_C, OTHER_G)], + False, + { + ("ga4gh:VA.partner", "measured"), + ("ga4gh:VA.other_c", "candidate"), + ("ga4gh:VA.other_g", "candidate"), + }, + False, + ), + # UTA unavailable: the record's prior links and candidate group are still this run's answer. + ( + [], + True, + { + ("ga4gh:VA.partner", "measured"), + ("ga4gh:VA.other_c", "candidate"), + ("ga4gh:VA.other_g", "candidate"), + }, + True, + ), + ], + ) + async def test_rerun_never_regroups_the_measured_allele( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + rerun_pairs, + rerun_unavailable, + expected_live, + candidate_group_kept, + ): + """The measured group belongs to the projection step, so whatever a rerun's candidates look like, the + authoritative link keeps pointing at the same measured group and is never re-versioned by RT. Only the + derived links and candidate groups are superseded, under the run's timestamp, unless the record's input + could not be asked about (UTA unavailable).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + translate = fake_translate( + { + MEASURED_C: CANONICAL_AUTHORITATIVE_DIGEST, + MEASURED_PARTNER_G: "ga4gh:VA.partner", + OTHER_C: "ga4gh:VA.other_c", + OTHER_G: "ga4gh:VA.other_g", + } + ) + first_construct = fake_construct({assay_hgvs: [(MEASURED_C, MEASURED_PARTNER_G), (OTHER_C, OTHER_G)]}) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", first_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + assert first.status == JobStatus.SUCCEEDED + + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + (first_candidate_group_id,) = { + link.projection_group + for link in _non_authoritative_links(session) + if link.projection_group != measured_group_id + } + first_live_ids = {link.id for link in _non_authoritative_links(session)} + + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + second_construct = ( + fake_construct({}, upstream_unavailable_by_hgvs={assay_hgvs: "could not connect to server"}) + if rerun_unavailable + else fake_construct({assay_hgvs: rerun_pairs}) + ) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", second_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, rerun) + + # The measured group and its authoritative link are exactly as the first run's projection left them. + assert _authoritative_link(session, record.id).projection_group == measured_group_id + assert _group(session, measured_group_id).current + assert len(_authoritative_versions(session, record.id)) == 2 + + live_derived = [link for link in _non_authoritative_links(session) if link.valid_to is None] + assert { + (link.allele.vrs_digest, "measured" if link.projection_group == measured_group_id else "candidate") + for link in live_derived + } == expected_live + + first_candidate_group = _group(session, first_candidate_group_id) + assert first_candidate_group.current is candidate_group_kept + if candidate_group_kept: + assert {link.id for link in live_derived} == first_live_ids + else: + # Every live candidate link points at a group minted by this run, under the run's timestamp. + live_candidate_groups = { + link.projection_group for link in live_derived if link.projection_group != measured_group_id + } + assert first_candidate_group_id not in live_candidate_groups + assert {_group(session, group_id).valid_from for group_id in live_candidate_groups} == { + first_candidate_group.valid_to + } + + async def test_a_coding_change_spelled_two_ways_stays_paired_with_its_genomic_projection( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """An insertion at an exon junction can be written on either side of it. Both spellings are one coding + allele, but only the one inside an exon projects to the genome; the allele joins that pair's group + rather than a coding-only group, whichever spelling the library lists first.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + junction_c = "NM_000001.1:c.10_11insGAG" + exon_c = "NM_000001.1:c.11_12insAGG" + exon_g = "NC_000001.11:g.2000_2001insCCT" + construct = fake_construct({"NM_000000.1:c.1A>G": [(junction_c, None), (exon_c, exon_g)]}) + translate = fake_translate({junction_c: "ga4gh:VA.shared_c", exon_c: "ga4gh:VA.shared_c", exon_g: "ga4gh:VA.g"}) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + candidate_groups: dict[int, set[str]] = {} + for link in _non_authoritative_links(session): + if link.valid_to is None and link.projection_group != measured_group_id: + candidate_groups.setdefault(link.projection_group, set()).add(link.allele.vrs_digest) + + assert list(candidate_groups.values()) == [{"ga4gh:VA.shared_c", "ga4gh:VA.g"}] + (group_id,) = candidate_groups + assert (_group(session, group_id).hgvs_c, _group(session, group_id).hgvs_g) == (exon_c, exon_g) + + async def test_a_changed_standalone_projection_detaches_the_measured_groups_rt_alleles( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Rewriting a measured group outside the reverse translation job retires the alleles RT added to it, + reports them as ``detached_rt_links``, and leaves candidate groups live.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + translate = fake_translate( + { + MEASURED_C: CANONICAL_AUTHORITATIVE_DIGEST, + MEASURED_PARTNER_G: "ga4gh:VA.partner", + OTHER_C: "ga4gh:VA.other_c", + OTHER_G: "ga4gh:VA.other_g", + } + ) + construct = fake_construct({"NM_000000.1:c.1A>G": [(MEASURED_C, MEASURED_PARTNER_G), (OTHER_C, OTHER_G)]}) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + record = _record_for(session, variant.id) + measured_group_id = _authoritative_link(session, record.id).projection_group + attached = [link for link in _non_authoritative_links(session) if link.projection_group == measured_group_id] + candidates = [link for link in _non_authoritative_links(session) if link.projection_group != measured_group_id] + assert attached and candidates + + class Shifted(FakeCoordinates): + def c_to_g(self, c_hgvs): + return "NC_000001.11:g.2000A>G" + + counts = project_measured_variants( + session, sample_score_set.id, coordinates=Shifted(), resolve_transcript=lambda *_: None + ) + session.commit() + + assert counts["detached_rt_links"] == len(attached) + for link in attached + candidates: + session.refresh(link) + assert all(link.valid_to is not None for link in attached) + assert all(link.valid_to is None for link in candidates) + assert not _group(session, measured_group_id).current + + @pytest.mark.parametrize( + "rerun_outcome, keeps_prior_link", + [ + ("upstream_unavailable", True), + ("translation_error", False), + ("not_translatable", False), + ("vrs_translation_failure", False), + ], + ) + async def test_rerun_keeps_prior_links_only_when_upstream_unavailable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + rerun_outcome, + keeps_prior_link, + ): + """A variant that fails a rerun because UTA was unavailable keeps the derived links from its last + successful run, since its inputs are unchanged. Any other failure is the rerun's answer for that + variant, so its prior links are retired.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", data={} + ) + variant_down = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#12", hgvs_nt="NM_000000.1:c.2G>T", data={} + ) + session.add_all([variant_ok, variant_down]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + ok_candidate = "NM_000001.1:c.5A>G" + down_candidate = "NM_000001.1:c.6G>T" + ids = {ok_candidate: "ga4gh:VA.ok", down_candidate: "ga4gh:VA.down"} + first_construct = fake_construct( + {"NM_000000.1:c.1A>G": [(ok_candidate, None)], "NM_000000.1:c.2G>T": [(down_candidate, None)]} + ) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", first_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate(ids)), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + assert first.status == JobStatus.SUCCEEDED + down_link = next( + link for link in _non_authoritative_links(session) if link.allele.vrs_digest == "ga4gh:VA.down" + ) + + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + ok_result = {"NM_000000.1:c.1A>G": [(ok_candidate, None)]} + down_message = {"NM_000000.1:c.2G>T": "reverse translation failed"} + second_translate = fake_translate(ids) + if rerun_outcome == "upstream_unavailable": + second_construct = fake_construct(ok_result, upstream_unavailable_by_hgvs=down_message) + elif rerun_outcome == "translation_error": + second_construct = fake_construct(ok_result, errors_by_hgvs=down_message) + elif rerun_outcome == "not_translatable": + second_construct = fake_construct(ok_result, not_translatable_by_hgvs=down_message) + else: + second_construct = first_construct + second_translate = fake_translate(ids, errors_by_hgvs={down_candidate: "cannot translate to VRS"}) + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", second_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", second_translate), + ): + second = await _reverse_translate(session, mock_worker_ctx, rerun) + assert second.status == JobStatus.SUCCEEDED + + session.refresh(down_link) + assert (down_link.valid_to is None) is keeps_prior_link + # The link's candidate group shares its fate. + assert _group(session, down_link.projection_group).current is keeps_prior_link + live_digests = sorted( + link.allele.vrs_digest for link in _non_authoritative_links(session) if link.valid_to is None + ) + assert live_digests == (["ga4gh:VA.down", "ga4gh:VA.ok"] if keeps_prior_link else ["ga4gh:VA.ok"]) + + async def test_all_failures_fail_the_job( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """If every variant's translation errors, the job fails and records FAILED annotations.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + construct = fake_construct({}, errors_by_hgvs={assay_hgvs: "forward translation failed"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0, "fold_in_missed": 0} + + assert _non_authoritative_links(session) == [] + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "forward translation failed" + # Library/input-level failures still carry the assay-level HGVS as metadata. + assert failed_events[0].event_metadata["hgvs_input"] == assay_hgvs + + async def test_partial_success_succeeds_with_mixed_annotations( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One variant translating and another erroring yields a SUCCEEDED job with both a + success and a failure annotation.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + variant_err = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#12", + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + ) + session.add_all([variant_ok, variant_err]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct( + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, + errors_by_hgvs={"NM_000000.1:c.2G>T": "no consequence"}, + ) + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} + + assert len(_cross_level_events(session, sample_score_set.id, disposition="present")) == 1 + assert len(_cross_level_events(session, sample_score_set.id, disposition="failed")) == 1 + assert len(_non_authoritative_links(session)) == 1 + + async def test_non_translatable_edit_is_skipped_not_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein consequence whose edit type has no DNA equivalence class (the library screens it + out as NOT_TRANSLATABLE) is a benign structural gap, not a failure: it is counted as skipped + and recorded as (not_applicable, not_translatable) — never in the failed tally. This is the + #767 fix: such variants must not pollute FAILED for the retroactive backfill.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + construct = fake_construct( + {}, + not_translatable_by_hgvs={ + assay_hgvs: "Protein edit type has no DNA equivalence class to reverse-translate" + }, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # A whole score set of non-translatable variants is a benign no-op, not a job failure. + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} + + # No alleles/links, and no failed event at all. + assert _non_authoritative_links(session) == [] + assert _cross_level_events(session, sample_score_set.id, disposition="failed") == [] + + # Recorded as a structural gap: not_applicable / not_translatable, carrying the input HGVS. + events = _cross_level_events(session, sample_score_set.id, reason="not_translatable") + assert len(events) == 1 + assert events[0].disposition == "not_applicable" + assert events[0].event_metadata["hgvs_input"] == assay_hgvs + + async def test_non_translatable_skip_coexists_with_success_and_failure( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Across three variants — one translatable, one non-translatable, one genuinely erroring — + each lands in its own bucket: translated / skipped / failed. The genuine error still records + FAILED (translation_error); the non-translatable one is a not_translatable skip.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", data={} + ) + variant_skip = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#12", hgvs_nt="NM_000000.1:c.2G>T", data={} + ) + variant_fail = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#13", hgvs_nt="NM_000000.1:c.3T>A", data={} + ) + session.add_all([variant_ok, variant_skip, variant_fail]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct( + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, + errors_by_hgvs={"NM_000000.1:c.3T>A": "forward translation failed"}, + not_translatable_by_hgvs={"NM_000000.1:c.2G>T": "no DNA equivalence class"}, + ) + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 1, "alleles_created": 1, "fold_in_missed": 1} + + # The genuine error is FAILED with reason translation_error — distinct from the benign skip. + failed = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed) == 1 + assert failed[0].reason == "translation_error" + assert failed[0].event_metadata["error_message"] == "forward translation failed" + + # The non-translatable one is not_applicable / not_translatable. + skipped = _cross_level_events(session, sample_score_set.id, reason="not_translatable") + assert len(skipped) == 1 + assert skipped[0].disposition == "not_applicable" + + async def test_upstream_outage_is_failed_as_api_error_not_translation_error( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A UTA outage that outlasts the library's retries is FAILED with reason api_error, so it + stays distinguishable from a genuine translation_error when auditing coverage gaps.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#11", hgvs_nt="NM_000000.1:c.1A>G", data={} + ) + variant_down = Variant( + score_set_id=sample_score_set.id, urn=f"{sample_score_set.urn}#12", hgvs_nt="NM_000000.1:c.2G>T", data={} + ) + session.add_all([variant_ok, variant_down]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + c_candidate = "NM_000001.1:c.5A>G" + outage = "reverse-translate-variants failed: server closed the connection unexpectedly" + construct = fake_construct( + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, + upstream_unavailable_by_hgvs={"NM_000000.1:c.2G>T": outage}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({c_candidate: "ga4gh:VA.coding"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} + + failed = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed) == 1 + assert failed[0].reason == "api_error" + assert failed[0].event_metadata["error_message"] == outage + assert _cross_level_events(session, sample_score_set.id, reason="translation_error") == [] + + async def test_partial_candidate_translation_failure_keeps_success_with_metadata( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One member of a projection pair translating and the other failing VRS translation leaves the + variant a SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + good_coding = "NM_000001.1:c.5A>G" + bad_genomic = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: [(good_coding, bad_genomic)]}) + translate = fake_translate( + {good_coding: "ga4gh:VA.coding"}, + errors_by_hgvs={bad_genomic: "untranslatable form"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 1} + assert len(_non_authoritative_links(session)) == 1 + + events = _cross_level_events(session, sample_score_set.id, disposition="present") + assert len(events) == 1 + failed_candidates = events[0].event_metadata["failed_candidates"] + assert failed_candidates == [{"hgvs": bad_genomic, "level": "genomic", "error": "untranslatable form"}] + + async def test_all_candidates_failing_translation_marks_variant_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When every candidate fails VRS translation, the variant is FAILED, no allele is + created, and the per-candidate errors are retained in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + bad_candidate = "NM_000001.1:c.9A>T" # a coding-only projection pair whose sole member fails + construct = fake_construct({assay_hgvs: [(bad_candidate, None)]}) + translate = fake_translate({}, errors_by_hgvs={bad_candidate: "untranslatable form"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0, "fold_in_missed": 0} + assert _non_authoritative_links(session) == [] + + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "All candidate HGVS failed VRS translation." + metadata = failed_events[0].event_metadata + assert metadata["hgvs_input"] == assay_hgvs + assert metadata["failed_candidates"] == [ + {"hgvs": bad_candidate, "level": "cdna", "error": "untranslatable form"} + ] + + async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-coding target aligned only at the genomic level has no resolvable coding + transcript (transcript_unresolved). The job tallies it in its `skipped` counter, but the + recoverable gap is recorded as a `failed` event — distinct from a hard translation + failure (reason `translation_failed`) and from a true biological absence + (no_coding_transcript → `absent`).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript to anchor on. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} + + # No translation attempted: no candidate alleles or links, and the variant is + # recorded as SKIPPED rather than FAILED. + assert _non_authoritative_links(session) == [] + # The job counts this as a skip, but a protein-coding target with no resolvable coding + # transcript is a *recoverable* gap (transcript_unresolved) → recorded as a `failed` event, + # distinct from a hard translation failure (which carries reason `translation_failed`). + unresolved = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(unresolved) == 1 + assert unresolved[0].disposition == "failed" + + async def test_genomic_accession_coding_target_is_reverse_translated( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A genomic-accession (NC_:g.) coding variant -- previously skipped for want of a + coding transcript -- is reverse-translated once the mapper emits a cdna + TargetGeneMapping, whose reference_accession anchors the projection.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity TargetGeneMapping (carrying the NM_); the + # genomic variant projects onto that transcript for reverse translation. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" + with ( + patch( + f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + ), + patch( + f"{RT_MODULE}.translate_hgvs_to_variation", + fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}), + ), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2, "fold_in_missed": 1} + events = _cross_level_events(session, sample_score_set.id) + assert events and all(e.disposition == "present" for e in events) + + def _run_mapped_date(self, session, target_gene_id): + """The mapped_date the (mock) mapping run stamped on its TargetGeneMappings.""" + return ( + session.query(TargetGeneMapping) + .filter( + TargetGeneMapping.target_gene_id == target_gene_id, + TargetGeneMapping.alignment_level == SequenceLevel.genomic, + ) + .first() + .mapped_date + ) + + async def test_uses_latest_cdna_row_within_the_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When a run has more than one cdna TargetGeneMapping for a target (same job_run_id), + RT reverse-translates against the newest (highest id), not an arbitrary one.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Mapping emits a cdna TargetGeneMapping (NM_999999.1) stamped with this run's job_run_id. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A newer cdna row (higher id) for the same target, from the same run. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=SequenceLevel.cdna, + reference_accession="NM_111111.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date, + job_run_id=sample_independent_variant_mapping_run.id, + ) + ) + session.commit() + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["transcripts"] = [inp.transcript for inp in inputs] + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.g"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + # The newest cdna row within the run wins, not the first-mapped one. + assert captured["transcripts"] == ["NM_111111.1"] + + async def test_ignores_stale_cdna_row_from_a_different_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A leftover cdna row that does not belong to the current run is not used: the current run + emitted no cdna row, so the variant is skipped (transcript unresolved) rather than + reverse-translated against the stale transcript. The record resolves its transcript by its own + run's job_run_id, so a row carrying no such anchor never leaks in (mavedb-api#763).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Current run maps genomic only -- no cdna row for this run's date. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A leftover cdna row from a *prior* run (an earlier date) must not be picked up. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=SequenceLevel.cdna, + reference_accession="NM_888888.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date - timedelta(days=1), + ) + ) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(skipped) == 1 + assert skipped[0].disposition == "failed" + + async def test_coding_target_skip_is_classified_recoverable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-coding target with no resolvable coding transcript is a *recoverable* + skip -- classified ``transcript_unresolved`` so it is findable, distinct from a + regulatory target's correct skip.""" + assert sample_score_set.target_genes[0].category == TargetCategory.protein_coding + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript resolves. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(skipped) == 1 + assert skipped[0].disposition == "failed" + + async def test_regulatory_target_skip_is_classified_correct( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A non-coding/regulatory target has no protein consequence: its skip is *correct*, + classified ``no_coding_transcript`` rather than the recoverable category.""" + target = sample_score_set.target_genes[0] + target.category = TargetCategory.regulatory + session.add(target) + session.commit() + + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0, "fold_in_missed": 0} + skipped = _cross_level_events(session, sample_score_set.id, reason="no_coding_transcript") + assert len(skipped) == 1 + assert skipped[0].disposition == "absent" + + async def test_translation_config_param_overrides_job_defaults( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A translation_config job param overrides the job's TranslationConfig defaults, and the + resulting config is passed through to construct_equivalent_variants.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + run = sample_independent_reverse_translation_run + run.job_params = {**run.job_params, "translation_config": {"wt_codon_mode": "unambiguous", "max_indel_size": 7}} + session.add(run) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + # Defaults the param did not override are preserved. + assert config.include_indels is True + + async def test_translation_config_defaults_when_param_absent( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no translation_config param the job falls back to its sensible defaults + (full codon equivalence class, indels included).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + async def test_protein_level_transcript_resolved_via_uta( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-only mapping has no cdna alignment transcript, so its coding transcript + is resolved NP_→NM_ via UTA and supplied as the VariantInput hint.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn=f"{sample_score_set.urn}#11", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: no cdna TargetGeneMapping, so the transcript must come from UTA. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + + assay_hgvs = "NP_000000.1:p.Met1Val" + c_candidate = "NM_000111.1:c.3A>G" + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) + + # Capture the transcript hint each VariantInput was built with, then delegate to the + # normal stub for the translation result. + captured_hints: dict[str, str | None] = {} + delegate = fake_construct({assay_hgvs: [(c_candidate, None)]}) + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured_hints.update({inp.hgvs: inp.transcript for inp in inputs}) + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000111.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1, "fold_in_missed": 0} + # The NP_ accession was resolved to its NM_ transcript and passed as the hint. + assert captured_hints == {assay_hgvs: "NM_000111.1"} diff --git a/tests/worker/lib/decorators/test_job_management.py b/tests/worker/lib/decorators/test_job_management.py index dd2e10ef4..fbfb370d8 100644 --- a/tests/worker/lib/decorators/test_job_management.py +++ b/tests/worker/lib/decorators/test_job_management.py @@ -16,7 +16,7 @@ from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus from mavedb.models.job_run import JobRun from mavedb.worker.lib.decorators.job_management import with_job_management from mavedb.worker.lib.managers.exceptions import JobStateError @@ -133,6 +133,67 @@ async def sample_error(ctx: dict, job_id: int, job_manager: JobManager): mock_start_job.assert_called_once() mock_error_job.assert_called_once() + async def test_decorator_marks_failed_and_reraises_on_cancellation( + self, session, mock_worker_ctx, mock_job_manager + ): + """A CancelledError (ARQ timeout or stall-recovery abort) must mark the row FAILED then re-raise. + + Re-raising is what lets ARQ record the cancellation as the job result so a concurrent + Job.abort() can confirm the job actually died. Without this handler the row would be + orphaned as RUNNING (CancelledError is a BaseException, not caught by `except Exception`). + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure") as mock_send_slack_job_failure, + patch.object(mock_job_manager, "start_job", return_value=None) as mock_start_job, + patch.object(mock_job_manager, "fail_job", return_value=None) as mock_fail_job, + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + mock_start_job.assert_called_once() + mock_fail_job.assert_called_once() + # The row is failed with a TIMEOUT category. + outcome = mock_fail_job.call_args.kwargs["result"] + assert outcome.status == JobStatus.FAILED + assert outcome.failure_category == FailureCategory.TIMEOUT + # A permanent-failure alert is emitted for the cancellation. + mock_send_slack_job_failure.assert_called_once() + + async def test_decorator_reraises_cancellation_even_if_marking_failed_errors( + self, session, mock_worker_ctx, mock_job_manager + ): + """If marking the cancelled job FAILED itself raises, the decorator still re-raises CancelledError. + + The salvage path must never swallow the cancellation — ARQ needs it to record the result. + """ + + @with_job_management + async def sample_cancel(ctx: dict, job_id: int, job_manager: JobManager): + raise asyncio.CancelledError() + + with ( + patch("mavedb.worker.lib.decorators.job_management.JobManager") as mock_job_manager_class, + patch("mavedb.worker.lib.decorators.job_management.send_slack_error") as mock_send_slack_error, + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_failure"), + patch.object(mock_job_manager, "start_job", return_value=None), + patch.object(mock_job_manager, "fail_job", side_effect=RuntimeError("db down")), + ): + mock_job_manager_class.return_value = mock_job_manager + + with pytest.raises(asyncio.CancelledError): + await sample_cancel(mock_worker_ctx, 999) + + # The inner failure is reported, and the original cancellation still propagates. + mock_send_slack_error.assert_called_once() + async def test_decorator_calls_start_job_and_skip_job_when_wrapped_function_returns_skipped_status( self, session, mock_worker_ctx, mock_job_manager ): diff --git a/tests/worker/lib/managers/test_job_manager.py b/tests/worker/lib/managers/test_job_manager.py index 980a30f02..84f709bf4 100644 --- a/tests/worker/lib/managers/test_job_manager.py +++ b/tests/worker/lib/managers/test_job_manager.py @@ -11,6 +11,7 @@ pytest.importorskip("arq") import re +from datetime import datetime, timedelta, timezone from unittest.mock import Mock, PropertyMock, patch from arq import ArqRedis @@ -1348,6 +1349,28 @@ def test_update_progress_success(self, session, arq_redis, with_populated_job_da assert job.progress_total == 100 assert job.progress_message == "Halfway done" + def test_update_progress_bumps_progress_updated_at_heartbeat( + self, session, arq_redis, with_populated_job_data, sample_job_run + ): + """Committing a progress update refreshes progress_updated_at via onupdate=now(). + + This is the liveness heartbeat the stalled-job sweeper reads: it must advance on every + committing progress checkpoint with no explicit code in the job. + """ + + manager = JobManager(session, arq_redis, sample_job_run.id) + + # Pin the heartbeat to a stale value so the bump is unambiguous. + stale = datetime.now(timezone.utc) - timedelta(hours=1) + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + job.progress_updated_at = stale + session.commit() + + manager.update_progress(10, 100, "tick", commit=True) # commit triggers the UPDATE + onupdate + + job = session.execute(select(JobRun).where(JobRun.id == sample_job_run.id)).scalar_one() + assert job.progress_updated_at > stale + def test_update_progress_success_does_not_overwrite_old_message_when_no_new_message_is_provided( self, session, arq_redis, with_populated_job_data, sample_job_run ): diff --git a/tests/worker/lib/managers/test_pipeline_manager.py b/tests/worker/lib/managers/test_pipeline_manager.py index 3028a31d2..d66a3a649 100644 --- a/tests/worker/lib/managers/test_pipeline_manager.py +++ b/tests/worker/lib/managers/test_pipeline_manager.py @@ -39,7 +39,7 @@ PipelineTransitionError, ) from mavedb.worker.lib.managers.pipeline_manager import PipelineManager -from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.lib.managers.utils import arq_job_id, arq_queue_name from tests.helpers.transaction_spy import TransactionSpy HANDLED_EXCEPTIONS_DURING_OBJECT_MANIPULATION = ( @@ -3567,6 +3567,7 @@ async def test_enqueue_in_arq_success(self, mock_pipeline_manager, retry, enqueu mock_job.id, _defer_by=datetime.timedelta(seconds=mock_job.retry_delay_seconds if retry else 0), _job_id=arq_job_id(mock_job), + _queue_name=arq_queue_name(mock_job), ) @pytest.mark.asyncio diff --git a/tests/worker/lib/managers/test_utils.py b/tests/worker/lib/managers/test_utils.py index 70e3ca24b..c1d9f27e5 100644 --- a/tests/worker/lib/managers/test_utils.py +++ b/tests/worker/lib/managers/test_utils.py @@ -4,6 +4,10 @@ pytest.importorskip("arq") +from unittest.mock import Mock + +from arq.constants import default_queue_name + from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import DependencyType, FailureCategory, JobStatus from mavedb.worker.lib.managers.constants import ( @@ -13,7 +17,10 @@ STARTABLE_JOB_STATUSES, TERMINAL_JOB_STATUSES, ) +from mavedb.models.job_run import JobRun from mavedb.worker.lib.managers.utils import ( + FAST_LANE_QUEUE_NAME, + arq_queue_name, classify_exception, construct_bulk_cancellation_result, job_dependency_is_met, @@ -148,3 +155,16 @@ def test_runtime_error_returns_unknown(self): def test_generic_exception_returns_unknown(self): assert classify_exception(Exception("generic")) == FailureCategory.UNKNOWN + + +@pytest.mark.unit +class TestArqQueueNameUnit: + """Tests for routing JobRuns to the fast-lane or default ARQ queue.""" + + @pytest.mark.parametrize("job_function", ["start_pipeline", "create_variants_for_score_set"]) + def test_fast_lane_functions_route_to_fast_lane_queue(self, job_function): + assert arq_queue_name(Mock(spec=JobRun, job_function=job_function)) == FAST_LANE_QUEUE_NAME + + @pytest.mark.parametrize("job_function", ["map_variants_for_score_set", "cleanup_stalled_jobs"]) + def test_other_functions_route_to_default_queue(self, job_function): + assert arq_queue_name(Mock(spec=JobRun, job_function=job_function)) == default_queue_name diff --git a/tests/worker/lib/test_projection.py b/tests/worker/lib/test_projection.py new file mode 100644 index 000000000..8c9d3d187 --- /dev/null +++ b/tests/worker/lib/test_projection.py @@ -0,0 +1,236 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.worker.lib.projection import MeasuredHgvs, project_measured_hgvs + + +class FakeCoordinates: + """Answers the projections in ``answers``; anything else raises, like hgvs on unknown input.""" + + def __init__(self, answers: dict[tuple[str, str], str]): + self.answers = answers + + def _answer(self, op: str, hgvs: str) -> str: + if (op, hgvs) not in self.answers: + raise ValueError(f"no {op} for {hgvs}") + return self.answers[(op, hgvs)] + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + return self._answer("g_to_c", g_hgvs) + + def normalize(self, hgvs_string: str) -> str: + return hgvs_string + + def c_to_g(self, c_hgvs: str) -> str: + return self._answer("c_to_g", c_hgvs) + + def c_to_p(self, c_hgvs: str) -> str: + return self._answer("c_to_p", c_hgvs) + + # Unnormalized projections default to the normalized answer; a test that needs them to differ adds a + # ``*_literal`` answer. + def c_to_g_literal(self, c_hgvs: str) -> str: + return self.answers.get(("c_to_g_literal", c_hgvs)) or self.c_to_g(c_hgvs) + + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + return self.answers.get(("g_to_c_literal", g_hgvs)) or self.g_to_c(g_hgvs, transcript) + + def c_to_n(self, c_hgvs: str) -> str: + return self.answers.get(("c_to_n", c_hgvs)) or c_hgvs.replace(":c.", ":n.") + + +MULTI_CODON_G = "NC_000008.11:g.95025143_95025145delinsAAC" +MULTI_CODON_C = "NM_152416.4:c.135_137delinsAAC" +INTRONIC_G = "NC_000002.12:g.214809442A>G" +INTRONIC_C = "NM_000465.4:c.123+5A>G" + + +@pytest.mark.unit +class TestProjectMeasuredHgvs: + def test_genomic_measurement_projects_to_coding_and_protein(self): + coordinates = FakeCoordinates( + { + ("g_to_c", MULTI_CODON_G): MULTI_CODON_C, + ("c_to_p", MULTI_CODON_C): "NP_689629.2:p.(Ser45_Val46delinsArgThr)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, MULTI_CODON_G, transcript="NM_152416.4", coordinates=coordinates + ) + + # A multi-codon change reverse translation cannot reach still gets its coding coordinate, and the + # prediction parens are stripped like reverse translation's. + assert measured == MeasuredHgvs( + hgvs_c=MULTI_CODON_C, hgvs_g=MULTI_CODON_G, hgvs_p="NP_689629.2:p.Ser45_Val46delinsArgThr" + ) + assert errors == [] + + def test_intronic_change_keeps_its_coding_coordinate_without_a_protein(self): + coordinates = FakeCoordinates({("g_to_c", INTRONIC_G): INTRONIC_C, ("c_to_p", INTRONIC_C): "NP_000456.2:p.?"}) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, INTRONIC_G, transcript="NM_000465.4", coordinates=coordinates + ) + + assert measured == MeasuredHgvs(hgvs_c=INTRONIC_C, hgvs_g=INTRONIC_G, hgvs_p=None) + assert errors == [] + + def test_coding_measurement_projects_to_genomic_and_protein(self): + coordinates = FakeCoordinates( + { + ("c_to_g", MULTI_CODON_C): MULTI_CODON_G, + ("c_to_p", MULTI_CODON_C): "NP_689629.2:p.(Ser45_Val46delinsArgThr)", + } + ) + + measured, _ = project_measured_hgvs(SequenceLevel.cdna, MULTI_CODON_C, transcript=None, coordinates=coordinates) + + assert measured.hgvs_g == MULTI_CODON_G + assert measured.hgvs_c == MULTI_CODON_C + + def test_genomic_measurement_without_a_transcript_has_only_its_genomic_hgvs(self): + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, MULTI_CODON_G, transcript=None, coordinates=FakeCoordinates({}) + ) + + assert measured == MeasuredHgvs(hgvs_g=MULTI_CODON_G) + assert errors == [] + + def test_failed_protein_keeps_the_coding_projection(self): + coordinates = FakeCoordinates({("g_to_c", MULTI_CODON_G): MULTI_CODON_C}) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, MULTI_CODON_G, transcript="NM_152416.4", coordinates=coordinates + ) + + assert measured == MeasuredHgvs(hgvs_c=MULTI_CODON_C, hgvs_g=MULTI_CODON_G) + assert len(errors) == 1 and errors[0].startswith("c_to_p") + + def test_protein_measurement_is_its_own_description(self): + measured, errors = project_measured_hgvs( + SequenceLevel.protein, "NP_689629.2:p.Val46Ala", transcript=None, coordinates=FakeCoordinates({}) + ) + + assert measured == MeasuredHgvs(hgvs_p="NP_689629.2:p.Val46Ala") + assert errors == [] + + def test_genomic_measurement_is_normalized_before_it_is_projected(self): + """An untrimmed delins at an exon boundary projects to the coding deletion it is, not to an + intron-spanning delins with no protein consequence.""" + raw = "NC_000010.11:g.87965286_87965289delinsG" + trimmed = "NC_000010.11:g.87965287_87965289del" + + class Normalizing(FakeCoordinates): + def normalize(self, hgvs_string: str) -> str: + return trimmed if hgvs_string == raw else hgvs_string + + coordinates = Normalizing( + { + ("g_to_c", trimmed): "NM_000314.8:c.1027_1029del", + ("c_to_p", "NM_000314.8:c.1027_1029del"): "NP_000305.3:p.(Val343del)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, raw, transcript="NM_000314.8", coordinates=coordinates + ) + + assert measured == MeasuredHgvs( + hgvs_c="NM_000314.8:c.1027_1029del", hgvs_g=trimmed, hgvs_p="NP_000305.3:p.Val343del" + ) + assert errors == [] + + def test_an_inversion_is_projected_as_the_equivalent_delins(self): + """VRS cannot represent ``inv``; hgvs's normalizing mapper writes one, so the coding projection is + taken from the unnormalized mapper instead.""" + genomic = "NC_000014.9:g.90403898_90403899delinsTG" + + class Literal(FakeCoordinates): + def g_to_c_literal(self, g_hgvs: str, transcript: str) -> str: + return "NM_006888.6:c.215_216delinsCA" + + coordinates = Literal( + { + ("g_to_c", genomic): "NM_006888.6:c.215_216inv", + ("c_to_p", "NM_006888.6:c.215_216delinsCA"): "NP_008819.1:p.(Met72Thr)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, genomic, transcript="NM_006888.6", coordinates=coordinates + ) + + assert measured.hgvs_c == "NM_006888.6:c.215_216delinsCA" + assert measured.hgvs_p == "NP_008819.1:p.Met72Thr" + assert errors == [] + + def test_a_coding_change_at_a_split_codon_has_no_genomic_projection(self): + """hgvs maps each end of the change separately, so its genomic form would delete the intron between.""" + split = "NM_000546.6:c.73_75delinsAGG" + coordinates = FakeCoordinates( + { + ("c_to_g", split): "NC_000017.11:g.7676403_7676522delinsCCT", + ("c_to_n", split): "NM_000546.6:n.215_217delinsAGG", + ("c_to_p", split): "NP_000537.3:p.(Leu25Arg)", + } + ) + + measured, errors = project_measured_hgvs(SequenceLevel.cdna, split, transcript=None, coordinates=coordinates) + + assert measured == MeasuredHgvs(hgvs_c=split, hgvs_g=None, hgvs_p="NP_000537.3:p.Leu25Arg") + assert errors == [] + + def test_a_reference_identity_is_projected_as_written(self): + """Normalizing ``g.100CCT=`` would collapse its three-base span to one position.""" + identity = "NC_000007.14:g.144548593_144548595=" + + class Collapsing(FakeCoordinates): + def normalize(self, hgvs_string: str) -> str: + return "NC_000007.14:g.144548593=" + + coordinates = Collapsing({("g_to_c", identity): "NM_022445.4:c.610_612="}) + + measured, _ = project_measured_hgvs( + SequenceLevel.genomic, identity, transcript="NM_022445.4", coordinates=coordinates + ) + + assert measured.hgvs_g == identity + assert measured.hgvs_c == "NM_022445.4:c.610_612=" + + def test_a_spanless_multi_base_identity_is_given_its_span(self): + """Legacy ``g.100CCT=`` names one position with a three-base reference, which hgvs rejects.""" + coordinates = FakeCoordinates( + { + ("g_to_c", "NC_000007.14:g.144548593_144548595="): "NM_022445.4:c.509_511=", + ("c_to_p", "NM_022445.4:c.509_511="): "NP_071890.2:p.(His170=)", + } + ) + + measured, errors = project_measured_hgvs( + SequenceLevel.genomic, "NC_000007.14:g.144548593CCT=", transcript="NM_022445.4", coordinates=coordinates + ) + + assert measured == MeasuredHgvs( + hgvs_c="NM_022445.4:c.509_511=", + hgvs_g="NC_000007.14:g.144548593_144548595=", + hgvs_p="NP_071890.2:p.His170=", + ) + assert errors == [] + + def test_a_synonymous_stop_is_written_with_equals(self): + coordinates = FakeCoordinates( + { + ("g_to_c", "NC_000002.12:g.214728677C>T"): "NM_000465.4:c.2333G>A", + ("c_to_p", "NM_000465.4:c.2333G>A"): "NP_000456.2:p.(Ter778Ter)", + } + ) + + measured, _ = project_measured_hgvs( + SequenceLevel.genomic, "NC_000002.12:g.214728677C>T", transcript="NM_000465.4", coordinates=coordinates + ) + + assert measured.hgvs_p == "NP_000456.2:p.Ter778=" diff --git a/tests/worker/lib/test_translation_ports.py b/tests/worker/lib/test_translation_ports.py new file mode 100644 index 000000000..ad874a009 --- /dev/null +++ b/tests/worker/lib/test_translation_ports.py @@ -0,0 +1,32 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from unittest.mock import MagicMock, patch + +from mavedb.worker.lib import translation_ports +from mavedb.worker.lib.translation_ports import uta_transcript_source + + +@pytest.mark.unit +class TestUtaTranscriptSource: + def test_retries_long_enough_to_ride_out_a_uta_outage(self, monkeypatch): + monkeypatch.setenv("UTA_DB_URL", "postgresql://anonymous@uta.example.org:5432/uta/uta_20241220") + client = MagicMock() + client.__enter__.return_value = client + + with patch.object(translation_ports.UtaClient, "from_url", return_value=client) as from_url: + with uta_transcript_source() as source: + assert source is client + + from_url.assert_called_once_with( + "postgresql://anonymous@uta.example.org:5432/uta/uta_20241220", max_attempts=6, backoff_seconds=2.0 + ) + + def test_requires_uta_db_url(self, monkeypatch): + monkeypatch.delenv("UTA_DB_URL", raising=False) + + with pytest.raises(RuntimeError, match="UTA_DB_URL must be set"): + with uta_transcript_source(): + pass diff --git a/tests/worker/test_settings.py b/tests/worker/test_settings.py new file mode 100644 index 000000000..b1209f6ce --- /dev/null +++ b/tests/worker/test_settings.py @@ -0,0 +1,31 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from arq.worker import get_kwargs + +from mavedb.worker.lib.managers.utils import FAST_LANE_JOB_FUNCTIONS, FAST_LANE_QUEUE_NAME +from mavedb.worker.settings import ArqFastLaneWorkerSettings, ArqWorkerSettings + + +@pytest.mark.unit +class TestArqFastLaneWorkerSettingsUnit: + """ARQ builds a Worker from the settings class ``__dict__``, so assert on what it actually reads.""" + + def test_consumes_fast_lane_queue(self): + assert get_kwargs(ArqFastLaneWorkerSettings)["queue_name"] == FAST_LANE_QUEUE_NAME + + def test_registers_exactly_the_fast_lane_functions(self): + functions = get_kwargs(ArqFastLaneWorkerSettings)["functions"] + assert {f.__name__ for f in functions} == FAST_LANE_JOB_FUNCTIONS + + def test_runs_no_cron_jobs(self): + assert "cron_jobs" not in get_kwargs(ArqFastLaneWorkerSettings) + + def test_main_worker_still_registers_fast_lane_functions(self): + """Fast-lane jobs already on the default queue at deploy time must still be runnable.""" + kwargs = get_kwargs(ArqWorkerSettings) + assert "queue_name" not in kwargs + assert FAST_LANE_JOB_FUNCTIONS <= {f.__name__ for f in kwargs["functions"]}