From b4a007b9434ed52b8397a42e97c34bae5d92ab3b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 4 Aug 2026 11:38:24 -0700 Subject: [PATCH 1/9] refactor(score-sets): extract CSV export logic into score_set_csv module Move all CSV-generation code for score set variant downloads out of score_sets.py into a new dedicated mavedb/lib/score_set_csv.py module, splitting it into pure functions (column planning, header assembly, row formatting) and DB-bound fetching to keep the logic testable in isolation. - Add parse_clinvar_namespace() to clinvar/utils.py, replacing the inline regex parsing that lived in score_sets.py - Add is_csv_output_null() to mave/utils.py to centralize null-value detection used when formatting CSV output - Move variant_to_csv_row, variants_to_csv_rows, get_score_set_variants_as_csv, and drop_na_columns_from_csv_file_rows into score_set_csv.py, and update routers/score_sets.py and scripts/export_public_data.py to import from the new location - Move corresponding tests into tests/lib/test_score_set_csv.py and add new unit tests for parse_clinvar_namespace and is_csv_output_null --- src/mavedb/lib/clinvar/constants.py | 6 + src/mavedb/lib/clinvar/utils.py | 19 +- src/mavedb/lib/mave/utils.py | 9 + src/mavedb/lib/score_set_csv.py | 484 +++++++++++++++++++++++ src/mavedb/lib/score_sets.py | 470 +--------------------- src/mavedb/routers/score_sets.py | 5 +- src/mavedb/scripts/export_public_data.py | 3 +- tests/lib/clinvar/test_utils.py | 20 + tests/lib/mave/__init__.py | 0 tests/lib/mave/test_utils.py | 31 ++ tests/lib/test_score_set.py | 152 ------- tests/lib/test_score_set_csv.py | 433 ++++++++++++++++++++ 12 files changed, 1009 insertions(+), 623 deletions(-) create mode 100644 src/mavedb/lib/score_set_csv.py create mode 100644 tests/lib/mave/__init__.py create mode 100644 tests/lib/mave/test_utils.py create mode 100644 tests/lib/test_score_set_csv.py diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index e70c4fee2..935376e34 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -1,8 +1,14 @@ import os +import re from pathlib import Path from urllib3.util.retry import Retry +CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +"""Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", +e.g. "clinvar.2024_01" for January 2024. +""" + TSV_VARIANT_ARCHIVE_BASE_URL = "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/archive" NCBI_REQUEST_HEADERS = { diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index 689e369ea..dcdef05d5 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -8,7 +8,7 @@ import sys from datetime import datetime from pathlib import Path -from typing import Dict +from typing import Dict, Optional import requests from filelock import FileLock @@ -17,6 +17,7 @@ from mavedb.lib.clinvar.constants import ( CLINVAR_CACHE_DIR, CLINVAR_FIELDS_TO_KEEP, + CLINVAR_NS_PATTERN, NCBI_REQUEST_HEADERS, NCBI_RETRY_STRATEGY, TSV_VARIANT_ARCHIVE_BASE_URL, @@ -28,6 +29,22 @@ logger = logging.getLogger(__name__) +def parse_clinvar_namespace(ns: str) -> Optional[str]: + """Parse a ClinVar-versioned namespace into its db_version string. + + Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` + for January 2024). The corresponding ``db_version`` stored in + ``clinical_controls`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). + + Returns ``None`` if *ns* does not match the expected pattern. + """ + m = CLINVAR_NS_PATTERN.match(ns) + if not m: + return None + year, month = m.group(1), m.group(2) + return f"{month}_{year}" + + def _ncbi_session() -> requests.Session: session = requests.Session() session.headers.update(NCBI_REQUEST_HEADERS) diff --git a/src/mavedb/lib/mave/utils.py b/src/mavedb/lib/mave/utils.py index dd6b75916..214532f37 100644 --- a/src/mavedb/lib/mave/utils.py +++ b/src/mavedb/lib/mave/utils.py @@ -31,3 +31,12 @@ def is_csv_null(value): if value == 0: return value return not value or NULL_VALUES_RE.fullmatch(str(value).strip().lower()) + + +_CSV_OUTPUT_NULL_RE = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) + + +def is_csv_output_null(value): + """Return True if a value should be replaced with the NA sentinel in CSV output.""" + value = str(value).strip().lower() + return _CSV_OUTPUT_NULL_RE.fullmatch(value) or not value diff --git a/src/mavedb/lib/score_set_csv.py b/src/mavedb/lib/score_set_csv.py new file mode 100644 index 000000000..b960833e7 --- /dev/null +++ b/src/mavedb/lib/score_set_csv.py @@ -0,0 +1,484 @@ +import csv +import io +from dataclasses import dataclass +from operator import attrgetter +from typing import Any, Callable, Iterable, List, Optional, Sequence + +from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy.orm import Session, aliased + +from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN +from mavedb.lib.clinvar.utils import parse_clinvar_namespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.mave.utils import NA_VALUE, is_csv_output_null +from mavedb.lib.validation.utilities import is_null as validate_is_null +from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class CsvColumnPlan: + namespaced_columns: dict[str, list[str]] + clinvar_namespaces: dict[str, str] + + +@dataclass +class CsvFetchResult: + variants: list[Variant] + mappings: Optional[list[Optional[MappedVariant]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + + +# --------------------------------------------------------------------------- +# Column-key resolvers for fixed-column namespaces +# --------------------------------------------------------------------------- + +_CORE_RESOLVERS: dict[str, Callable] = { + "hgvs_nt": attrgetter("hgvs_nt"), + "hgvs_pro": attrgetter("hgvs_pro"), + "hgvs_splice": attrgetter("hgvs_splice"), + "accession": attrgetter("urn"), +} + +_VEP_RESOLVERS: dict[str, Callable] = { + "vep_functional_consequence": lambda mapping: mapping.vep_functional_consequence if mapping else None, +} + +_GNOMAD_RESOLVERS: dict[str, Callable] = { + "gnomad_af": lambda gnomad_data: gnomad_data.allele_frequency if gnomad_data else None, +} + +_CLINGEN_RESOLVERS: dict[str, Callable] = { + "clingen_allele_id": lambda mapping: mapping.clingen_allele_id if mapping else None, +} + +_CLINVAR_RESOLVERS: dict[str, Callable] = { + "clinical_significance": attrgetter("clinical_significance"), + "clinical_review_status": attrgetter("clinical_review_status"), +} + + +def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: + """Return the string representation of *value*, or *na_rep* if the value is None.""" + if is_csv_output_null(value): + return na_rep + return str(value) + + +def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: + """Shared key-formatting logic used by both header assembly and row assembly.""" + # ClinVar columns are always namespaced to differentiate versions, even if the user has requested un-namespaced output. + if CLINVAR_NS_PATTERN.match(namespace): + return f"{namespace}.{column_key}" + + # The "core" namespace is always un-namespaced, even if the user has requested namespaced output. + if namespace == "core": + return column_key + + # All other namespaces are namespaced if the user has requested namespaced output, and un-namespaced otherwise. + if namespaced: + return f"{namespace}.{column_key}" + + return column_key + + +def _custom_columns(dataset_columns: dict, col_name: str) -> list[str]: + return [col for col in [str(x) for x in list(dataset_columns.get(col_name, []))]] + + +# --------------------------------------------------------------------------- +# Pure functions +# --------------------------------------------------------------------------- + + +def plan_csv_columns( + dataset_columns: dict, + namespaces: list[str], + *, + include_custom_columns: bool = True, + include_post_mapped_hgvs: bool = False, +) -> CsvColumnPlan: + """Build the namespaced column map and ClinVar namespace mapping.""" + namespaced_score_set_columns: dict[str, list[str]] = { + "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], + "mavedb": [], + } + + if include_post_mapped_hgvs: + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") + namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") + + for namespace in namespaces: + namespaced_score_set_columns[namespace] = [] + + if "scores" in namespaced_score_set_columns: + if include_custom_columns: + # the required score column is transitively included via the _custom_columns function. + namespaced_score_set_columns["scores"] = _custom_columns(dataset_columns, "score_columns") + else: + namespaced_score_set_columns["scores"] = [REQUIRED_SCORE_COLUMN] + if "counts" in namespaced_score_set_columns: + if include_custom_columns: + namespaced_score_set_columns["counts"] = _custom_columns(dataset_columns, "count_columns") + if "vep" in namespaced_score_set_columns: + namespaced_score_set_columns["vep"].append("vep_functional_consequence") + if "gnomad" in namespaced_score_set_columns: + namespaced_score_set_columns["gnomad"].append("gnomad_af") + if "clingen" in namespaced_score_set_columns: + namespaced_score_set_columns["clingen"].append("clingen_allele_id") + + clinvar_namespaces: dict[str, str] = {} + for ns in namespaces: + db_version = parse_clinvar_namespace(ns) + if db_version is not None: + clinvar_namespaces[ns] = db_version + namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] + + return CsvColumnPlan( + namespaced_columns=namespaced_score_set_columns, + clinvar_namespaces=clinvar_namespaces, + ) + + +def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: + """Build the flat column-header list from the namespace dict.""" + return [ + _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols + ] + + +# --------------------------------------------------------------------------- +# Row assembly +# --------------------------------------------------------------------------- + + +def variant_to_csv_row( + variant: Variant, + columns: dict[str, list[str]], + mapping: Optional[MappedVariant] = None, + gnomad_data: Optional[GnomADVariant] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> dict[str, Any]: + """Format a variant into a dict containing the keys specified in *columns*.""" + row: dict[str, Any] = {} + + for column_key in columns.get("core", []): + resolver = _CORE_RESOLVERS.get(column_key) + if resolver is None: + raise ValueError(f"unrecognized core column: {column_key}") + + value = str(resolver(variant)) + row[column_key] = _value_or_na(value, na_rep) + + for column_key in columns.get("mavedb", []): + if column_key == "post_mapped_hgvs_g": + value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep + if value == na_rep: + fallback_hgvs = ( + get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + ) + if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): + value = fallback_hgvs + else: + value = na_rep + + elif column_key == "post_mapped_hgvs_p": + value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep + if value == na_rep: + fallback_hgvs = ( + get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + ) + if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): + value = fallback_hgvs + else: + value = na_rep + + elif column_key == "post_mapped_hgvs_c": + value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep + elif column_key == "post_mapped_hgvs_at_assay_level": + value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep + elif column_key == "post_mapped_vrs_digest": + digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + value = digest if digest is not None else na_rep + else: + raise ValueError(f"unrecognized mavedb column: {column_key}") + + row[_format_column_key("mavedb", column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + for ns in ("vep", "gnomad", "clingen"): + resolvers = {"vep": _VEP_RESOLVERS, "gnomad": _GNOMAD_RESOLVERS, "clingen": _CLINGEN_RESOLVERS}[ns] + source = {"vep": mapping, "gnomad": gnomad_data, "clingen": mapping}[ns] + for column_key in columns.get(ns, []): + resolver = resolvers.get(column_key) + if resolver is None: + raise ValueError(f"unrecognized {ns} column: {column_key}") + value = resolver(source) + row[_format_column_key(ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + for data_ns in ("scores", "counts"): + data_key = f"{data_ns[:-1]}_data" + parent = variant.data.get(data_key) if variant.data else None + for column_key in columns.get(data_ns, []): + value = str(parent.get(column_key)) if parent else na_rep + row[_format_column_key(data_ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + for namespace_key, namespace_cols in columns.items(): + if not CLINVAR_NS_PATTERN.match(namespace_key): + continue + clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) + for column_key in namespace_cols: + resolver = _CLINVAR_RESOLVERS.get(column_key) + if resolver is None: + raise ValueError(f"unrecognized clinvar column: {column_key}") + value = str(resolver(clinvar_entry)) if clinvar_entry else na_rep + row[_format_column_key(namespace_key, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + return row + + +def variants_to_csv_rows( + variants: Sequence[Variant], + columns: dict[str, list[str]], + mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> Iterable[dict[str, Any]]: + """Format each variant into a dictionary row containing the keys specified in *columns*.""" + n = len(variants) + _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n + _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n + ) + return map( + lambda t: variant_to_csv_row( + t[0], + columns, + mapping=t[1], + gnomad_data=t[2], + clinvar_data_by_ns=t[3], + namespaced=namespaced, + na_rep=na_rep, + ), + zip(variants, _mappings, _gnomad, _clinvar), + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def drop_na_columns_from_csv_file_rows( + rows_data: Iterable[dict[str, Any]], columns: list[str] +) -> tuple[list[dict[str, Any]], list[str]]: + """Process rows_data for downloadable CSV by removing empty columns.""" + rows_data = list(rows_data) + columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + columns_to_remove = [] + + for col in columns_to_check: + if all(validate_is_null(row[col]) for row in rows_data): + columns_to_remove.append(col) + for row in rows_data: + row.pop(col, None) + + columns = [col for col in columns if col not in columns_to_remove] + return rows_data, columns + + +# --------------------------------------------------------------------------- +# DB-bound fetching +# --------------------------------------------------------------------------- + + +def fetch_variant_csv_data( + db: Session, + score_set: ScoreSet, + namespaced_columns: dict[str, list[str]], + clinvar_namespaces: dict[str, str], + *, + include_post_mapped_hgvs: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, +) -> CsvFetchResult: + """Fetch variant data from the database for CSV generation.""" + namespaces = list(namespaced_columns.keys()) + + need_mappings = ( + include_post_mapped_hgvs + or "clingen" in namespaces + or "vep" in namespaces + or "gnomad" in namespaces + or bool(clinvar_namespaces) + ) + need_gnomad = "gnomad" in namespaces + + variants: list[Variant] = [] + mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None + gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + + select_columns: list[Any] = [Variant] + if need_mappings: + select_columns.append(MappedVariant) + if need_gnomad: + select_columns.append(GnomADVariant) + + query = ( + select(*select_columns) + .where(Variant.score_set_id == score_set.id) + .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) + ) + + if need_mappings: + query = query.join( + MappedVariant, + and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), + isouter=True, + ) + + if need_gnomad: + query = query.join( + MappedVariant.gnomad_variants.of_type(GnomADVariant), + isouter=True, + ).where( + or_( + and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), + GnomADVariant.id.is_(None), + ) + ) + + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + result = db.execute(query).all() + + for row in result: + variant = row[0] + variants.append(variant) + + if need_mappings and mappings is not None: + mappings.append(row[1]) + + if need_gnomad and gnomad_data_list is not None: + idx = 2 if need_mappings else 1 + gnomad_data_list.append(row[idx]) + + clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + if clinvar_namespaces and mappings is not None: + mv_ids = [m.id for m in mappings if m is not None] + for ns, db_version in clinvar_namespaces.items(): + mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + if mv_ids: + aliased_cc = aliased(ClinicalControl) + cc_query = ( + select( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id, + aliased_cc, + ) + .join( + aliased_cc, + mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, + ) + .where( + and_( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), + aliased_cc.db_name == "ClinVar", + aliased_cc.db_version == db_version, + ) + ) + ) + for mv_id, cc in db.execute(cc_query).all(): + mv_to_cc[mv_id] = cc + clinvar_data_map[ns] = mv_to_cc + + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + if clinvar_namespaces and mappings is not None: + clinvar_per_variant = [] + for mapping in mappings: + row_clinvar: dict[str, Optional[ClinicalControl]] = {} + for ns, mv_to_cc in clinvar_data_map.items(): + if mapping is not None and mapping.id is not None: + row_clinvar[ns] = mv_to_cc.get(mapping.id) + else: + row_clinvar[ns] = None + clinvar_per_variant.append(row_clinvar) + + return CsvFetchResult( + variants=variants, + mappings=mappings, + gnomad_data=gnomad_data_list, + clinvar_per_variant=clinvar_per_variant, + ) + + +# --------------------------------------------------------------------------- +# Public composer +# --------------------------------------------------------------------------- + + +def get_score_set_variants_as_csv( + db: Session, + score_set: ScoreSet, + namespaces: List[str], + namespaced: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, + drop_na_columns: Optional[bool] = None, + include_custom_columns: Optional[bool] = True, + include_post_mapped_hgvs: Optional[bool] = False, +) -> str: + """Get the variant data from a score set as a CSV string.""" + assert type(score_set.dataset_columns) is dict + + plan = plan_csv_columns( + score_set.dataset_columns, + namespaces, + include_custom_columns=bool(include_custom_columns), + include_post_mapped_hgvs=bool(include_post_mapped_hgvs), + ) + + fetched = fetch_variant_csv_data( + db, + score_set, + plan.namespaced_columns, + plan.clinvar_namespaces, + include_post_mapped_hgvs=bool(include_post_mapped_hgvs), + start=start, + limit=limit, + ) + + rows_data = variants_to_csv_rows( + fetched.variants, + columns=plan.namespaced_columns, + namespaced=namespaced, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + ) + + rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) + + if drop_na_columns: + rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) + + stream = io.StringIO() + writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) + writer.writeheader() + writer.writerows(rows_data) + return stream.getvalue() diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 8e3c8debb..698bc515a 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -1,17 +1,14 @@ -import csv -import io import logging -import re from collections import Counter, defaultdict from operator import attrgetter -from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence +from typing import TYPE_CHECKING, BinaryIO, Optional, Sequence import numpy as np import pandas as pd from pandas.testing import assert_index_equal -from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload +from sqlalchemy import and_, func, or_, select from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -19,7 +16,6 @@ HGVS_NT_COLUMN, HGVS_PRO_COLUMN, HGVS_SPLICE_COLUMN, - REQUIRED_SCORE_COLUMN, VARIANT_COUNT_DATA, VARIANT_SCORE_DATA, ) @@ -27,8 +23,6 @@ from mavedb.lib.permissions import Action, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list -from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -38,9 +32,6 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier @@ -57,7 +48,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant -from mavedb.view_models.search import ScoreSetsSearch, ControlledKeywordFilterOption +from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: from mavedb.lib.permissions import Action @@ -66,10 +57,6 @@ logger = logging.getLogger(__name__) -# Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", -# e.g. "clinvar.2024_01" for January 2024. -CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") - class HGVSColumns: NUCLEOTIDE: str = "hgvs_nt" # dataset.constants.hgvs_nt_column @@ -587,258 +574,6 @@ def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) ) -def get_score_set_variants_as_csv( - db: Session, - score_set: ScoreSet, - namespaces: List[str], - namespaced: Optional[bool] = None, - start: Optional[int] = None, - limit: Optional[int] = None, - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = True, - include_post_mapped_hgvs: Optional[bool] = False, -) -> str: - """ - Get the variant data from a score set as a CSV string. - - Parameters - __________ - db : Session - The database session to use. - score_set : ScoreSet - The score set to get the variants from. - namespaces : List[str] - The namespaces for data: "scores", "counts", "vep", "gnomad", "clingen", and/or - ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" - for January 2024, which joins on db_name="ClinVar" and db_version="01_2024"). - namespaced: Optional[bool] = None - Whether namespace the columns or not. - start : int, optional - The index to start from. If None, starts from the beginning. - limit : int, optional - The maximum number of variants to return. If None, returns all variants. - drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. - include_custom_columns : bool, optional - Whether to include custom columns defined in the score set. Defaults to True. - include_post_mapped_hgvs : bool, optional - Whether to include post-mapped HGVS notations and VEP functional consequence in the output. Defaults to False. If True, the output will include - columns for post-mapped HGVS genomic (g.) and protein (p.) notations, and VEP functional consequence. - - Returns - _______ - str - The CSV string containing the variant data. - """ - assert type(score_set.dataset_columns) is dict - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if include_custom_columns: - if "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] - ] - if "counts" in namespaced_score_set_columns: - namespaced_score_set_columns["counts"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] - ] - elif "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"].append(REQUIRED_SCORE_COLUMN) - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - # Parse ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH". - # The corresponding db_version stored in clinical_controls is "MONTH_YEAR". - clinvar_namespaces: dict[str, str] = {} # namespace -> db_version (MONTH_YEAR) - for ns in namespaces: - m = CLINVAR_NS_PATTERN.match(ns) - if m: - year, month = m.group(1), m.group(2) - db_version = f"{month}_{year}" - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) - ) - need_gnomad = "gnomad" in namespaces - - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) - ) - - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data is not None: - idx = 2 if need_mappings else 1 - gnomad_data.append(row[idx]) - - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinicalControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - - rows_data = variants_to_csv_rows( - variants, - columns=namespaced_score_set_columns, - namespaced=namespaced, - mappings=mappings, - gnomad_data=gnomad_data, - clinvar_data_by_ns=clinvar_per_variant, - ) # type: ignore - - rows_columns = [] - for namespace, cols in namespaced_score_set_columns.items(): - for col in cols: - if CLINVAR_NS_PATTERN.match(namespace): - # ClinVar versioned namespaces always include the full namespace prefix - # to avoid column-name collisions when multiple versions are requested. - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace not in ["core", "mavedb"]: - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace == "mavedb": - rows_columns.append(f"mavedb.{col}") - else: - rows_columns.append(col) - - if drop_na_columns: - rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) - - stream = io.StringIO() - writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) - writer.writeheader() - writer.writerows(rows_data) - return stream.getvalue() - - -def drop_na_columns_from_csv_file_rows( - rows_data: Iterable[dict[str, Any]], columns: list[str] -) -> tuple[list[dict[str, Any]], list[str]]: - """Process rows_data for downloadable CSV by removing empty columns.""" - # Convert map to list. - rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - columns_to_remove = [] - - # Check if all values in a column are None or "NA" - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): - columns_to_remove.append(col) - for row in rows_data: - row.pop(col, None) # Remove column from each row - - # Remove these columns from the header list - columns = [col for col in columns if col not in columns_to_remove] - return rows_data, columns - - -null_values_re = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) - - -def is_null(value): - """Return True if a string represents a null value.""" - value = str(value).strip().lower() - return null_values_re.fullmatch(value) or not value - - def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: """ Return True if the IntegrityError was caused by the unique constraint on score_set.replaces_id. @@ -852,203 +587,6 @@ def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: return "replaces_id" in detail -def variant_to_csv_row( - variant: Variant, - columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, - gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> dict[str, Any]: - """ - Format a variant into a containing the keys specified in `columns`. - - Parameters - ---------- - variant : variant.models.Variant - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mapping : variant.models.MappedVariant, optional - Mapped variant corresponding to the variant. - gnomad_data : variant.models.GnomADVariant, optional - gnomAD variant data corresponding to the variant. - clinvar_data_by_ns : dict[str, Optional[ClinicalControl]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - dict[str, Any] - """ - row: dict[str, Any] = {} - # Handle each column key explicitly as part of its namespace. - for column_key in columns.get("core", []): - if column_key == "hgvs_nt": - value = str(variant.hgvs_nt) - elif column_key == "hgvs_pro": - value = str(variant.hgvs_pro) - elif column_key == "hgvs_splice": - value = str(variant.hgvs_splice) - elif column_key == "accession": - value = str(variant.urn) - if is_null(value): - value = na_rep - - # export columns in the `core` namespace without a namespace - row[column_key] = value - for column_key in columns.get("mavedb", []): - if column_key == "post_mapped_hgvs_g": - value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_p": - value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_c": - value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep - elif column_key == "post_mapped_hgvs_at_assay_level": - value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep - elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep - if is_null(value): - value = na_rep - key = f"mavedb.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("vep", []): - if column_key == "vep_functional_consequence": - vep_functional_consequence = mapping.vep_functional_consequence if mapping else None - if vep_functional_consequence is not None: - value = vep_functional_consequence - else: - value = na_rep - key = f"vep.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("scores", []): - parent = variant.data.get("score_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"scores.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("counts", []): - parent = variant.data.get("count_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"counts.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("gnomad", []): - if column_key == "gnomad_af": - gnomad_af = gnomad_data.allele_frequency if gnomad_data else None - if gnomad_af is not None: - value = str(gnomad_af) - else: - value = na_rep - key = f"gnomad.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("clingen", []): - if column_key == "clingen_allele_id": - clingen_allele_id = mapping.clingen_allele_id if mapping else None - if clingen_allele_id is not None: - value = str(clingen_allele_id) - else: - value = na_rep - key = f"clingen.{column_key}" if namespaced else column_key - row[key] = value - # Handle ClinVar-versioned namespaces (e.g. "clinvar.2024_01"). - # These always use the full "namespace.column" key regardless of the namespaced flag - # to avoid collisions when multiple versions are requested. - for namespace_key, namespace_cols in columns.items(): - if not CLINVAR_NS_PATTERN.match(namespace_key): - continue - clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) - for column_key in namespace_cols: - if column_key == "clinical_significance": - value = str(clinvar_entry.clinical_significance) if clinvar_entry else na_rep - elif column_key == "clinical_review_status": - value = str(clinvar_entry.clinical_review_status) if clinvar_entry else na_rep - else: - value = na_rep - if is_null(value): - value = na_rep - row[f"{namespace_key}.{column_key}"] = value - return row - - -def variants_to_csv_rows( - variants: Sequence[Variant], - columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, - gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> Iterable[dict[str, Any]]: - """ - Format each variant into a dictionary row containing the keys specified in `columns`. - - Parameters - ---------- - variants : list[variant.models.Variant] - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mappings : list[Optional[variant.models.MappedVariant]], optional - List of mapped variants corresponding to the variants. - gnomad_data : list[Optional[variant.models.GnomADVariant]], optional - List of gnomAD variant data corresponding to the variants. - clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinicalControl]]]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - list[dict[str, Any]] - """ - n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n - _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( - clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n - ) - return map( - lambda t: variant_to_csv_row( - t[0], - columns, - mapping=t[1], - gnomad_data=t[2], - clinvar_data_by_ns=t[3], - namespaced=namespaced, - na_rep=na_rep, - ), - zip(variants, _mappings, _gnomad, _clinvar), - ) - - def find_meta_analyses_for_score_sets(db: Session, urns: list[str]) -> list[ScoreSet]: """ Find all score sets that are meta-analyses for a specified collection of other score sets. diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 4d1a30ad5..1f3aeec29 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -53,16 +53,15 @@ from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration +from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN +from mavedb.lib.score_set_csv import get_score_set_variants_as_csv, variants_to_csv_rows from mavedb.lib.score_sets import ( - CLINVAR_NS_PATTERN, csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, get_current_mapped_variants_for_annotation, - get_score_set_variants_as_csv, is_replaces_id_unique_violation, refresh_variant_urns, - variants_to_csv_rows, ) from mavedb.lib.score_sets import ( search_score_sets as _search_score_sets, diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 3ce31a2ba..b390aadeb 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -28,7 +28,8 @@ from mavedb.lib.annotation.annotate import variant_highest_level_annotation from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv +from mavedb.lib.score_set_csv import get_score_set_variants_as_csv +from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License diff --git a/tests/lib/clinvar/test_utils.py b/tests/lib/clinvar/test_utils.py index 082c55586..257402d38 100644 --- a/tests/lib/clinvar/test_utils.py +++ b/tests/lib/clinvar/test_utils.py @@ -9,6 +9,7 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import ( fetch_clinvar_variant_data, + parse_clinvar_namespace, validate_clinvar_variant_summary_date, ) @@ -45,6 +46,25 @@ def _make_gzipped_tsv(text: str) -> bytes: ) +@pytest.mark.unit +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", "01_2024"), + ("clinvar.2015_12", "12_2015"), + ("clinvar.2026_06", "06_2026"), + ("clinvar.2024_00", None), + ("clinvar.2024_13", None), + ("scores", None), + ("clinvar", None), + ("clinvar.2024_01.extra", None), + ("", None), + ], +) +def test_parse_clinvar_namespace(ns, expected): + assert parse_clinvar_namespace(ns) == expected + + @pytest.mark.unit class TestValidateClinvarVariantSummaryDate: def test_valid_past_date(self): diff --git a/tests/lib/mave/__init__.py b/tests/lib/mave/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/mave/test_utils.py b/tests/lib/mave/test_utils.py new file mode 100644 index 000000000..44716595b --- /dev/null +++ b/tests/lib/mave/test_utils.py @@ -0,0 +1,31 @@ +import pytest + +from mavedb.lib.mave.utils import is_csv_output_null + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value, expected", + [ + (None, True), + ("", True), + (" ", True), + ("NA", True), + ("na", True), + ("None", True), + ("none", True), + ("NaN", True), + ("nan", True), + ("null", True), + ("NULL", True), + ("nil", True), + ("N/A", True), + ("undefined", True), + ("1.5", False), + ("0", False), + ("hello", False), + ("p.Met1Val", False), + ], +) +def test_is_csv_output_null(value, expected): + assert bool(is_csv_output_null(value)) is expected diff --git a/tests/lib/test_score_set.py b/tests/lib/test_score_set.py index 3ca40d4a6..53d1e874b 100644 --- a/tests/lib/test_score_set.py +++ b/tests/lib/test_score_set.py @@ -22,7 +22,6 @@ create_variants_data, csv_data_to_df, fetch_score_set_search_filter_options, - variant_to_csv_row, ) from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import ( @@ -556,154 +555,3 @@ def test_fetch_score_set_search_filter_options_with_no_permitted_score_sets(setu "publication_db_names": [], "publication_journals": [], } - - -class MockVariant: - """Lightweight mock for Variant used in variant_to_csv_row tests.""" - - def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): - self.urn = urn - self.hgvs_nt = hgvs_nt - self.hgvs_splice = hgvs_splice - self.hgvs_pro = hgvs_pro - self.data = data - - -class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" - - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "1.5" - - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "42" - - def test_score_data_with_custom_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, na_rep="N/A") - - assert row["score"] == "N/A" - - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): - variant = MockVariant( - hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", - data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, - ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" diff --git a/tests/lib/test_score_set_csv.py b/tests/lib/test_score_set_csv.py new file mode 100644 index 000000000..9ec913090 --- /dev/null +++ b/tests/lib/test_score_set_csv.py @@ -0,0 +1,433 @@ +import pytest + +from mavedb.lib.score_set_csv import ( + assemble_csv_headers, + drop_na_columns_from_csv_file_rows, + plan_csv_columns, + variant_to_csv_row, +) + +# --------------------------------------------------------------------------- +# MockVariant +# --------------------------------------------------------------------------- + + +class MockVariant: + """Lightweight mock for Variant used in variant_to_csv_row tests.""" + + def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): + self.urn = urn + self.hgvs_nt = hgvs_nt + self.hgvs_splice = hgvs_splice + self.hgvs_pro = hgvs_pro + self.data = data + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowNullHandling +# --------------------------------------------------------------------------- + + +class TestVariantToCsvRowNullHandling: + """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" + + def test_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"score_data": {}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_score_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"count_data": {}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_count_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_score_data_with_valid_value_preserved(self): + variant = MockVariant(data={"score_data": {"score": 1.5}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "1.5" + + def test_count_data_with_valid_value_preserved(self): + variant = MockVariant(data={"count_data": {"count1": 42}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "42" + + def test_score_data_with_custom_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, na_rep="N/A") + + assert row["score"] == "N/A" + + def test_namespaced_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["scores.score"] == "NA" + + def test_namespaced_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["counts.count1"] == "NA" + + def test_core_columns_with_none_hgvs_uses_na_rep(self): + variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") + columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "NA" + assert row["hgvs_pro"] == "NA" + assert row["hgvs_splice"] == "NA" + assert row["accession"] == "urn:mavedb:00000001-a-1#1" + + def test_mixed_columns_with_missing_data(self): + variant = MockVariant( + hgvs_nt="g.1A>G", + hgvs_pro="p.Met1Val", + data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, + ) + columns = { + "core": ["hgvs_nt", "hgvs_pro"], + "scores": ["score", "se"], + "counts": ["count1", "count2"], + } + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "g.1A>G" + assert row["hgvs_pro"] == "p.Met1Val" + assert row["score"] == "NA" + assert row["se"] == "0.1" + assert row["count1"] == "NA" + assert row["count2"] == "5" + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowUnrecognizedKey +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespace, columns", + [ + ("core", {"core": ["bogus_col"]}), + ("mavedb", {"mavedb": ["bogus_col"]}), + ("vep", {"vep": ["bogus_col"]}), + ("gnomad", {"gnomad": ["bogus_col"]}), + ("clingen", {"clingen": ["bogus_col"]}), + ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), + ], +) +def test_unrecognized_column_key_raises(namespace, columns): + variant = MockVariant() + with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): + variant_to_csv_row(variant, columns) + + +# --------------------------------------------------------------------------- +# TestPlanCsvColumns +# --------------------------------------------------------------------------- + + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar", + [ + # scores-only + ( + ["scores"], + {}, + {"core", "mavedb", "scores"}, + ["score", "se", "epsilon"], + {}, + ), + # counts-only + ( + ["counts"], + {}, + {"core", "mavedb", "counts"}, + None, + {}, + ), + # both scores and counts + ( + ["scores", "counts"], + {}, + {"core", "mavedb", "scores", "counts"}, + ["score", "se", "epsilon"], + {}, + ), + # vep adds its column + ( + ["vep"], + {}, + {"core", "mavedb", "vep"}, + None, + {}, + ), + # gnomad adds its column + ( + ["gnomad"], + {}, + {"core", "mavedb", "gnomad"}, + None, + {}, + ), + # clingen adds its column + ( + ["clingen"], + {}, + {"core", "mavedb", "clingen"}, + None, + {}, + ), + # include_custom_columns=False -> only REQUIRED_SCORE_COLUMN for scores + ( + ["scores"], + {"include_custom_columns": False}, + {"core", "mavedb", "scores"}, + ["score"], + {}, + ), + # include_post_mapped_hgvs populates mavedb namespace + ( + ["scores"], + {"include_post_mapped_hgvs": True}, + {"core", "mavedb", "scores"}, + ["score", "se", "epsilon"], + {}, + ), + # single ClinVar namespace + ( + ["clinvar.2024_01"], + {}, + {"core", "mavedb", "clinvar.2024_01"}, + None, + {"clinvar.2024_01": "01_2024"}, + ), + # multiple ClinVar versions + ( + ["clinvar.2024_01", "clinvar.2025_06"], + {}, + {"core", "mavedb", "clinvar.2024_01", "clinvar.2025_06"}, + None, + {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, + ), + ], +) +def test_plan_csv_columns(namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces, **kwargs) + + assert set(plan.namespaced_columns.keys()) == expected_ns_keys + assert plan.clinvar_namespaces == expected_clinvar + + if expected_score_cols is not None: + assert plan.namespaced_columns["scores"] == expected_score_cols + + # core always has the standard 4 columns + assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] + + # vep, gnomad, clingen get their fixed columns when present + if "vep" in plan.namespaced_columns: + assert plan.namespaced_columns["vep"] == ["vep_functional_consequence"] + if "gnomad" in plan.namespaced_columns: + assert plan.namespaced_columns["gnomad"] == ["gnomad_af"] + if "clingen" in plan.namespaced_columns: + assert plan.namespaced_columns["clingen"] == ["clingen_allele_id"] + + # ClinVar namespaces get their standard columns + for ns in expected_clinvar: + assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] + + +def test_plan_csv_columns_post_mapped_hgvs_populates_mavedb(): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"], include_post_mapped_hgvs=True) + assert plan.namespaced_columns["mavedb"] == [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_digest", + ] + + +# --------------------------------------------------------------------------- +# TestAssembleCsvHeaders +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaced_columns, namespaced, expected", + [ + # Unnamespaced: flat column names + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, + False, + ["accession", "hgvs_nt", "score", "se"], + ), + # Namespaced: scores get prefix, core does not + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, + True, + ["accession", "hgvs_nt", "scores.score"], + ), + # mavedb namespace always gets prefix when namespaced + ( + {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, + True, + ["accession", "mavedb.post_mapped_hgvs_g"], + ), + # ClinVar namespaces always get prefix regardless of namespaced flag + ( + {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, + False, + ["accession", "clinvar.2024_01.clinical_significance"], + ), + # Mixed: respects insertion order + ( + { + "core": ["accession"], + "mavedb": [], + "scores": ["score"], + "clinvar.2024_01": ["clinical_significance"], + }, + True, + ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], + ), + # Empty mavedb namespace when not namespaced produces nothing + ( + {"core": ["hgvs_nt"], "mavedb": []}, + False, + ["hgvs_nt"], + ), + ], +) +def test_assemble_csv_headers(namespaced_columns, namespaced, expected): + assert assemble_csv_headers(namespaced_columns, namespaced) == expected + + +# --------------------------------------------------------------------------- +# TestDropNaColumns +# --------------------------------------------------------------------------- + + +class TestDropNaColumns: + def test_removes_all_na_hgvs_column(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert "hgvs_splice" not in new_cols + assert "hgvs_nt" in new_cols + assert "hgvs_pro" in new_cols + for row in new_rows: + assert "hgvs_splice" not in row + + def test_keeps_column_with_some_values(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + def test_does_not_touch_non_hgvs_columns(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert "score" in new_cols + assert "hgvs_splice" not in new_cols + + def test_empty_rows_does_not_crash(self): + rows = [] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert new_rows == [] + assert new_cols == [] From 1493cc09e8a3228da6d01584e314d8675052ebeb Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 10:37:14 -0700 Subject: [PATCH 2/9] refactor(csv)!: replace score_set_csv with a namespaced CSV package Split the monolithic score_set_csv.py into mavedb/lib/csv/, a package organized by responsibility (namespaces, columns, entries, fetch, annotations, specs, deprecated_params) instead of one file mixing column planning, row formatting, and DB fetching for a single record type. score_set.py and variant.py sit on top as the two record-specific entry points, sharing the namespace/column machinery beneath them. This is the foundation for variant-level CSV export and namespace discovery (following commits): the old module had no way to answer "what does this record have data for" or to serve a single variant, so extending it in place would have meant bolting both concepts onto code that already conflated planning and fetching. - Introduce namespace discovery as first-class: CsvNamespace enum, CsvNamespaceGroup, and available_score_set_csv_namespaces / available_variant_csv_namespaces, replacing the hand-maintained CLINVAR_NS_PATTERN validity check in the router - Add lib/annotation/flatten.py to project a variant's VA-Spec interpretation onto scalar CSV columns, sharing acmg.py's new acmg_evidence_outcome_code() with the VA-Spec evidence-line builder so the two representations can't drift - Thread an optional containing_classification_ids set through functional_classification_of_variant and pathogenicity_classification_of_variant, resolving the long-standing TODOs about O(1) membership checks when classifying many variants - Move parse_clinvar_namespace (clinvar/utils.py) and CSV-null-output detection (mave/utils.py) into the csv package, since both were export-side concerns living outside it - Add drop_unused_hgvs_columns, scores_custom, and score_set namespaces; keep drop_na_columns, include_post_mapped_hgvs, and include_custom_columns working as deprecated aliases via resolve_deprecated_csv_params - Add title to ShorterScoreSet so namespace-discovery responses can label a calibration's owning score set without a second lookup BREAKING CHANGE: score_set_csv.py is removed. Callers importing get_score_set_variants_as_csv or variants_to_csv_rows from it must import from mavedb.lib.csv.score_set / mavedb.lib.csv.columns instead. --- src/mavedb/lib/acmg.py | 42 + src/mavedb/lib/annotation/classification.py | 35 +- src/mavedb/lib/annotation/evidence_line.py | 12 +- src/mavedb/lib/annotation/flatten.py | 140 +++ src/mavedb/lib/clinvar/constants.py | 6 - src/mavedb/lib/clinvar/utils.py | 19 +- src/mavedb/lib/csv/__init__.py | 0 src/mavedb/lib/csv/annotations.py | 110 ++ src/mavedb/lib/csv/columns.py | 252 ++++ src/mavedb/lib/csv/deprecated_params.py | 106 ++ src/mavedb/lib/csv/entries.py | 192 +++ src/mavedb/lib/csv/fetch.py | 176 +++ src/mavedb/lib/csv/namespaces.py | 254 ++++ src/mavedb/lib/csv/score_set.py | 124 ++ src/mavedb/lib/csv/specs.py | 254 ++++ src/mavedb/lib/csv/variant.py | 388 +++++++ src/mavedb/lib/mave/utils.py | 14 +- src/mavedb/lib/score_set_csv.py | 484 -------- src/mavedb/lib/urns.py | 57 + src/mavedb/lib/validation/urn_re.py | 4 + src/mavedb/view_models/csv_namespace.py | 43 + src/mavedb/view_models/score_set.py | 19 +- tests/lib/annotation/test_flatten.py | 160 +++ tests/lib/clinvar/test_utils.py | 20 - tests/lib/csv/__init__.py | 0 tests/lib/csv/test_columns.py | 622 ++++++++++ tests/lib/csv/test_entries.py | 64 + tests/lib/csv/test_namespaces.py | 242 ++++ tests/lib/csv/test_specs.py | 113 ++ tests/lib/csv/test_variant.py | 1160 +++++++++++++++++++ tests/lib/mave/test_utils.py | 31 - tests/lib/test_acmg.py | 34 + tests/lib/test_score_set_csv.py | 433 ------- tests/lib/test_urns.py | 99 ++ 34 files changed, 4682 insertions(+), 1027 deletions(-) create mode 100644 src/mavedb/lib/annotation/flatten.py create mode 100644 src/mavedb/lib/csv/__init__.py create mode 100644 src/mavedb/lib/csv/annotations.py create mode 100644 src/mavedb/lib/csv/columns.py create mode 100644 src/mavedb/lib/csv/deprecated_params.py create mode 100644 src/mavedb/lib/csv/entries.py create mode 100644 src/mavedb/lib/csv/fetch.py create mode 100644 src/mavedb/lib/csv/namespaces.py create mode 100644 src/mavedb/lib/csv/score_set.py create mode 100644 src/mavedb/lib/csv/specs.py create mode 100644 src/mavedb/lib/csv/variant.py delete mode 100644 src/mavedb/lib/score_set_csv.py create mode 100644 src/mavedb/view_models/csv_namespace.py create mode 100644 tests/lib/annotation/test_flatten.py create mode 100644 tests/lib/csv/__init__.py create mode 100644 tests/lib/csv/test_columns.py create mode 100644 tests/lib/csv/test_entries.py create mode 100644 tests/lib/csv/test_namespaces.py create mode 100644 tests/lib/csv/test_specs.py create mode 100644 tests/lib/csv/test_variant.py delete mode 100644 tests/lib/mave/test_utils.py delete mode 100644 tests/lib/test_score_set_csv.py create mode 100644 tests/lib/test_urns.py diff --git a/src/mavedb/lib/acmg.py b/src/mavedb/lib/acmg.py index d7de860e8..d786e6a96 100644 --- a/src/mavedb/lib/acmg.py +++ b/src/mavedb/lib/acmg.py @@ -8,6 +8,48 @@ from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +def acmg_evidence_outcome_code(criterion: str, evidence_strength: Optional[str]) -> str: + """Build the ACMG 2015 evidence outcome code for a criterion and the strength it was met at. + + Three rules, which are the ACMG convention rather than anything MaveDB invented: + + - no strength means the criterion was evaluated and *not* met, written ``"PS3_not_met"`` + - STRONG is the criterion's baseline, so it is written bare: ``"PS3"`` + - any other strength is suffixed: ``"PS3_moderate"`` + + Takes the criterion code and strength *name* as strings rather than enums so that the VA-Spec + annotation builders and the flat exports can share one implementation despite drawing their + enumerations from different places. That also means this survives any future decision about which + enumeration is canonical. + + Parameters + ---------- + criterion : str + The criterion code, e.g. ``"PS3"`` or ``"BS3"``. + evidence_strength : Optional[str] + The strength name, e.g. ``"MODERATE"``. None when the criterion was not met. + + Returns + ------- + str + The evidence outcome code. + + Examples + -------- + >>> acmg_evidence_outcome_code("PS3", "STRONG") + 'PS3' + >>> acmg_evidence_outcome_code("PS3", "MODERATE") + 'PS3_moderate' + >>> acmg_evidence_outcome_code("BS3", None) + 'BS3_not_met' + """ + if evidence_strength is None: + return f"{criterion}_not_met" + if evidence_strength.upper() == StrengthOfEvidenceProvided.STRONG.name: + return criterion + return f"{criterion}_{evidence_strength.lower()}" + + def points_evidence_strength_equivalent( points: int, ) -> tuple[Optional[ACMGCriterion], Optional[StrengthOfEvidenceProvided]]: diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index 08fc3b208..c7707eb1c 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -21,13 +21,35 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): INDETERMINATE = "indeterminate" +def _classification_contains_variant( + functional_classification: ScoreCalibrationFunctionalClassification, + mapped_variant: MappedVariant, + containing_classification_ids: Optional[set[int]], +) -> bool: + """Whether this classification's score range contains the variant. + + Prefers a pre-resolved id set, which is an O(1) check. Falls back to the ORM relationship, which is + correct but loads every variant of the range. + """ + if containing_classification_ids is not None: + return functional_classification.id in containing_classification_ids + return mapped_variant.variant in functional_classification.variants + + def functional_classification_of_variant( - mapped_variant: MappedVariant, score_calibration: ScoreCalibration + mapped_variant: MappedVariant, + score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: """Classify a variant's functional impact as normal, abnormal, or indeterminate. Uses the primary score calibration and its functional ranges. Raises ValueError if required calibration or score is missing. + + *containing_classification_ids*, when given, is the set of functional-classification ids already known + to contain this variant. Pass it to avoid the ORM membership check below, which loads every variant of + every range. A caller classifying many variants should resolve membership once from the association + table; see ``mavedb.lib.csv.variant``. """ if not mapped_variant.variant.score_set.score_calibrations: raise ValueError( @@ -41,11 +63,8 @@ def functional_classification_of_variant( " Unable to classify functional impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in functional_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for functional_range in score_calibration.functional_classifications: - if mapped_variant.variant in functional_range.variants: + if _classification_contains_variant(functional_range, mapped_variant, containing_classification_ids): if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -58,6 +77,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( mapped_variant: MappedVariant, score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[ Optional[ScoreCalibrationFunctionalClassification], VariantPathogenicityEvidenceLine.Criterion, @@ -87,11 +107,8 @@ def pathogenicity_classification_of_variant( " Unable to classify clinical impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in pathogenicity_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for pathogenicity_range in score_calibration.functional_classifications: - if mapped_variant.variant in pathogenicity_range.variants: + if _classification_contains_variant(pathogenicity_range, mapped_variant, containing_classification_ids): if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index 8ebf7f163..d30e90542 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -10,8 +10,8 @@ StudyResult, VariantPathogenicityProposition, ) -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided +from mavedb.lib.acmg import acmg_evidence_outcome_code from mavedb.lib.annotation.classification import ( functional_classification_of_variant, pathogenicity_classification_of_variant, @@ -44,16 +44,14 @@ def acmg_evidence_line( mapped_variant, score_calibration ) + evidence_outcome_code = acmg_evidence_outcome_code( + evidence_outcome.value, evidence_strength.name if evidence_strength else None + ) + if not evidence_strength: - evidence_outcome_code = f"{evidence_outcome.value}_not_met" strength_of_evidence = None direction_of_evidence = Direction.NEUTRAL else: - evidence_outcome_code = ( - f"{evidence_outcome.value}_{evidence_strength.name.lower()}" - if evidence_strength != StrengthOfEvidenceProvided.STRONG - else evidence_outcome.value - ) strength_of_evidence = MappableConcept( primaryCoding=Coding( code=evidence_strength, diff --git a/src/mavedb/lib/annotation/flatten.py b/src/mavedb/lib/annotation/flatten.py new file mode 100644 index 000000000..04d2506f7 --- /dev/null +++ b/src/mavedb/lib/annotation/flatten.py @@ -0,0 +1,140 @@ +"""Flatten a variant's VA-Spec clinical interpretation into scalar values. + +The rest of this package builds nested VA-Spec structures, which are faithful to the standard but not +consumable by a spreadsheet. This projects the same classification onto flat scalars. +""" + +from dataclasses import dataclass +from typing import Optional + +from ga4gh.va_spec.acmg_2015 import AcmgClassification + +from mavedb.lib.acmg import acmg_evidence_outcome_code +from mavedb.lib.annotation.classification import ( + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration + + +@dataclass(frozen=True) +class FlatAnnotation: + """A single variant's clinical interpretation under one calibration, flattened to scalars. + + A field is ``None`` when the calibration cannot support it; exporters render that as NA. + """ + + functional_classification: Optional[str] = None + """"normal", "abnormal", or "indeterminate".""" + + acmg_criterion: Optional[str] = None + """The ACMG 2015 criterion evaluated, e.g. "PS3" or "BS3".""" + + acmg_evidence_strength: Optional[str] = None + """Strength of evidence, e.g. "MODERATE" or "MODERATE_PLUS"; None when the criterion was not met. + + MaveDB's own enumeration, finer-grained than VA-Spec's: an M+ range reports MODERATE_PLUS here while + its VA-Spec annotation must report moderate. + """ + + acmg_evidence_outcome_code: Optional[str] = None + """ACMG evidence outcome code, e.g. "PS3_moderate", "PS3" (strong), or "BS3_not_met".""" + + pathogenicity_classification: Optional[str] = None + """"PATHOGENIC", "BENIGN", or "UNCERTAIN_SIGNIFICANCE".""" + + calibration_urn: Optional[str] = None + calibration_title: Optional[str] = None + + research_use_only: Optional[bool] = None + """Whether the calibration is marked research use only. + + Carried so a consumer holding only exported rows can tell that a criterion came from thresholds never + validated for clinical use. + """ + + +def flatten_annotation( + mapped_variant: MappedVariant, + score_calibration: Optional[ScoreCalibration], + containing_classification_ids: Optional[set[int]] = None, +) -> FlatAnnotation: + """Flatten a variant's clinical interpretation under *score_calibration* into scalar values. + + A calibration with ranges but no ACMG classifications yields a functional classification only, matching + the annotation layer. Evidence strength uses MaveDB's own enumeration, so MODERATE_PLUS is preserved. + + Args: + containing_classification_ids: forwarded to the classifiers. Supply it when flattening many + variants; the fallback loads every variant of every score range. + + Returns: + An all-``None`` annotation when *score_calibration* is None. A calibration that exists but defines + no ranges reports its identity and standing with no interpretation. + """ + # No calibration means nothing to say under this namespace (e.g. it belongs to another score set). + if score_calibration is None: + return FlatAnnotation() + + # Rangeless: reporting identity distinguishes "defines no ranges" from "no calibration applies here". + if not score_calibration.functional_classifications: + return FlatAnnotation( + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + _, functional_classification = functional_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + functional_only = FlatAnnotation( + functional_classification=functional_classification.value, + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + # No ACMG classification on any range: stop at the functional classification rather than reporting a + # not-met PS3 the curator never asserted. + if all(fc.acmg_classification is None for fc in score_calibration.functional_classifications): + return functional_only + + # VA-Spec strength deliberately unused: it has already collapsed MODERATE_PLUS to MODERATE. + containing_range, criterion, _va_spec_evidence_strength = pathogenicity_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + # Read the strength off the containing range, which keeps MODERATE_PLUS. Reachable in practice: + # `points_evidence_strength_equivalent` assigns M+ to +/-3 point ranges (e.g. the Excalibr + # calibrations). + # + # TODO(#XXX): move the lossy MODERATE_PLUS -> MODERATE conversion to the VA-Spec boundary instead of + # `classification.py`, upstream of every consumer; this special case then disappears. + native_evidence_strength = ( + containing_range.acmg_classification.evidence_strength + if containing_range is not None and containing_range.acmg_classification is not None + else None + ) + evidence_strength_name = native_evidence_strength.name if native_evidence_strength is not None else None + + # `pathogenicity_classification_of_variant` returns PS3 even for variants in no range, so the range, + # not the criterion, tells us whether evidence exists. No strength means evaluated and not met. + if containing_range is None or evidence_strength_name is None: + pathogenicity_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE + elif criterion.name.startswith("B"): + pathogenicity_classification = AcmgClassification.BENIGN + else: + pathogenicity_classification = AcmgClassification.PATHOGENIC + + return FlatAnnotation( + functional_classification=functional_only.functional_classification, + acmg_criterion=criterion.value, + acmg_evidence_strength=evidence_strength_name, + acmg_evidence_outcome_code=acmg_evidence_outcome_code(criterion.value, evidence_strength_name), + pathogenicity_classification=pathogenicity_classification.name, + calibration_urn=functional_only.calibration_urn, + calibration_title=functional_only.calibration_title, + research_use_only=functional_only.research_use_only, + ) diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index 935376e34..e70c4fee2 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -1,14 +1,8 @@ import os -import re from pathlib import Path from urllib3.util.retry import Retry -CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") -"""Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", -e.g. "clinvar.2024_01" for January 2024. -""" - TSV_VARIANT_ARCHIVE_BASE_URL = "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/archive" NCBI_REQUEST_HEADERS = { diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index dcdef05d5..689e369ea 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -8,7 +8,7 @@ import sys from datetime import datetime from pathlib import Path -from typing import Dict, Optional +from typing import Dict import requests from filelock import FileLock @@ -17,7 +17,6 @@ from mavedb.lib.clinvar.constants import ( CLINVAR_CACHE_DIR, CLINVAR_FIELDS_TO_KEEP, - CLINVAR_NS_PATTERN, NCBI_REQUEST_HEADERS, NCBI_RETRY_STRATEGY, TSV_VARIANT_ARCHIVE_BASE_URL, @@ -29,22 +28,6 @@ logger = logging.getLogger(__name__) -def parse_clinvar_namespace(ns: str) -> Optional[str]: - """Parse a ClinVar-versioned namespace into its db_version string. - - Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` - for January 2024). The corresponding ``db_version`` stored in - ``clinical_controls`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). - - Returns ``None`` if *ns* does not match the expected pattern. - """ - m = CLINVAR_NS_PATTERN.match(ns) - if not m: - return None - year, month = m.group(1), m.group(2) - return f"{month}_{year}" - - def _ncbi_session() -> requests.Session: session = requests.Session() session.headers.update(NCBI_REQUEST_HEADERS) diff --git a/src/mavedb/lib/csv/__init__.py b/src/mavedb/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py new file mode 100644 index 000000000..5f5736e8e --- /dev/null +++ b/src/mavedb/lib/csv/annotations.py @@ -0,0 +1,110 @@ +"""Resolving each row's calibration interpretations for a CSV export. + +Shared by both exports, and kept out of ``columns`` because filling these cells needs the database. +""" + +from typing import Callable, Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.lib.csv.entries import visible_calibrations +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) +from mavedb.models.variant import Variant + + +def calibrations_for_namespaces( + db: Session, + calibration_namespaces: dict[str, str], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> dict[str, ScoreCalibration]: + """Load the calibrations named by the requested namespaces, keyed by namespace. + + Looked up by the URN the caller named, not by what a score set offers: the namespace *is* the request. + Which means this, not discovery, is the gate — naming a private calibration's URN directly must not + serve its interpretation, so *may_read_calibration* is applied here too. + """ + if not calibration_namespaces: + return {} + + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.urn.in_(list(calibration_namespaces.values()))) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ) + ) + ).all() + + by_urn = { + str(calibration.urn): calibration for calibration in visible_calibrations(calibrations, may_read_calibration) + } + return {namespace: by_urn[urn] for namespace, urn in calibration_namespaces.items() if urn in by_urn} + + +def containing_classification_ids(db: Session, variant_ids: Sequence[int]) -> dict[int, set[int]]: + """Map each variant to the score-classification ids whose range contains it. + + One query over the association table, replacing the ORM membership check in + ``mavedb.lib.annotation.classification`` that loads every variant of every range once per range per + row — the dominant cost of these exports at score-set scale. + """ + if not variant_ids: + return {} + + membership: dict[int, set[int]] = {variant_id: set() for variant_id in variant_ids} + rows = db.execute( + select( + score_calibration_functional_classification_variants_association_table.c.variant_id, + score_calibration_functional_classification_variants_association_table.c.functional_classification_id, + ).where(score_calibration_functional_classification_variants_association_table.c.variant_id.in_(variant_ids)) + ).all() + for variant_id, classification_id in rows: + membership[variant_id].add(classification_id) + + return membership + + +def annotations_for_rows( + db: Session, + variants: Sequence[Variant], + mappings: Sequence[Optional[MappedVariant]], + calibration_namespaces: dict[str, str], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: + """Flatten every row's interpretation under each requested calibration namespace. + + A calibration from a different score set than the row leaves that namespace empty: a score from one + assay carries no meaning under another's thresholds. So does one the caller may not read. + + Returns: + None when no calibration namespace was requested, so the caller can skip the work entirely. + """ + if not calibration_namespaces: + return None + + calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, may_read_calibration) + # TODO(#372): non-null id fields + membership = containing_classification_ids(db, [variant.id for variant in variants]) # type: ignore + + rows: list[dict[str, Optional[FlatAnnotation]]] = [] + for variant, mapping in zip(variants, mappings): + # TODO(#372): non-null id fields + contained = membership.get(variant.id, set()) # type: ignore + annotations: dict[str, Optional[FlatAnnotation]] = {} + for namespace in calibration_namespaces: + calibration = calibrations_by_ns.get(namespace) + if mapping is None or calibration is None or calibration.score_set_id != variant.score_set_id: + annotations[namespace] = None + else: + annotations[namespace] = flatten_annotation(mapping, calibration, contained) + rows.append(annotations) + + return rows diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py new file mode 100644 index 000000000..7853a1d73 --- /dev/null +++ b/src/mavedb/lib/csv/columns.py @@ -0,0 +1,252 @@ +"""Column planning and row assembly for the CSV exports. + +Pure functions over already-fetched objects; nothing here touches the database. What a namespace *is* +lives in ``specs``; this module only applies it. +""" + +import csv +import io +from dataclasses import dataclass +from typing import Any, Iterable, Optional, Sequence + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.namespaces import ( + CALIBRATION_NS_PATTERN, + CLINVAR_NS_PATTERN, + parse_calibration_namespace, + parse_clinvar_namespace, +) +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from mavedb.lib.mave.utils import NA_VALUE +from mavedb.lib.validation.utilities import is_null as validate_is_null +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +_OUTPUT_NULL_STRINGS = frozenset({"none", "nan", "na", "undefined", "n/a", "null", "nil"}) + + +@dataclass(frozen=True) +class CsvColumnPlan: + namespaced_columns: dict[str, list[str]] + """Namespace -> list of column keys to emit for that namespace.""" + clinvar_namespaces: dict[str, str] + """Requested ClinVar namespace -> the ``"MM_YYYY"`` db_version it names.""" + calibration_namespaces: dict[str, str] + """Requested calibration namespace -> the calibration URN it names.""" + + +def _is_output_null(value: Any) -> bool: + """Whether *value* should be written as the NA sentinel rather than rendered. + + Distinct from ``lib.mave.utils.is_csv_null``, which decides whether a value read *from* an uploaded + file counts as missing: that one copes with pandas NA types and treats 0 specially. + """ + text = str(value).strip().lower() + return not text or text in _OUTPUT_NULL_STRINGS + + +def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: + """Return the string representation of *value*, or *na_rep* if the value is null-ish.""" + if _is_output_null(value): + return na_rep + return str(value) + + +def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: + """Shared key-formatting logic used by both header assembly and row assembly.""" + # Always namespaced regardless of the caller's preference: the release or calibration URN is what + # disambiguates columns when several are requested at once. + if CLINVAR_NS_PATTERN.match(namespace) or CALIBRATION_NS_PATTERN.match(namespace): + return f"{namespace}.{column_key}" + + if namespace == CORE_NAMESPACE: # core is never namespaced + return column_key + + if namespaced: + spec = namespace_spec(namespace) + prefix = spec.emit_under if spec is not None and spec.emit_under is not None else namespace + return f"{prefix}.{column_key}" + + return column_key + + +def plan_csv_columns(dataset_columns: dict, namespaces: list[str]) -> CsvColumnPlan: + """Build the namespaced column map and the ClinVar and calibration namespace mappings. + + An unknown namespace is kept with no columns rather than rejected — validating the vocabulary belongs + to the request layer. + """ + namespaced_columns: dict[str, list[str]] = {} + clinvar_namespaces: dict[str, str] = {} + calibration_namespaces: dict[str, str] = {} + + for namespace in dict.fromkeys([CORE_NAMESPACE, *namespaces]): + spec = namespace_spec(namespace) + namespaced_columns[namespace] = spec.columns(dataset_columns) if spec else [] + + db_version = parse_clinvar_namespace(namespace) + if db_version is not None: + clinvar_namespaces[namespace] = db_version + + calibration_urn = parse_calibration_namespace(namespace) + if calibration_urn is not None: + calibration_namespaces[namespace] = calibration_urn + + return CsvColumnPlan( + namespaced_columns=namespaced_columns, + clinvar_namespaces=clinvar_namespaces, + calibration_namespaces=calibration_namespaces, + ) + + +def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: + """Build the flat column-header list from the namespace dict. + + Raises: + ValueError: if two namespaces resolve to the same header. Un-namespaced output strips the prefix + that would otherwise keep them apart, so requesting two namespaces that share a column name + would emit it twice; the callers that ask for un-namespaced output request one namespace each, + and this holds them to it rather than letting a future caller find out from a broken file. + """ + headers = [ + _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols + ] + + duplicates = sorted({header for header in headers if headers.count(header) > 1}) + if duplicates: + raise ValueError( + f"CSV namespaces resolve to duplicate columns: {', '.join(duplicates)}." + f" Requested namespaces: {', '.join(namespaced_columns)}." + ) + + return headers + + +def variant_to_csv_row( + variant: Variant, + columns: dict[str, list[str]], + mapping: Optional[MappedVariant] = None, + gnomad_data: Optional[GnomADVariant] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + annotations_by_ns: Optional[dict[str, Optional[FlatAnnotation]]] = None, + match_type: Optional[str] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> dict[str, Any]: + """Format a variant into a dict containing the keys specified in *columns*. + + Args: + clinvar_data_by_ns, annotations_by_ns: per-row data for the parameterized namespaces, keyed by + requested namespace. A namespace with no entry renders as *na_rep*. + """ + row: dict[str, Any] = {} + + # Built once per row, not per namespace: a 100k-variant export with ten namespaces would otherwise + # build this dict, and walk `variant.data` twice, a million times over. + row_sources: dict[RowSource, Any] = { + RowSource.VARIANT: variant, + RowSource.MAPPING: mapping, + RowSource.GNOMAD: gnomad_data, + RowSource.MATCH_TYPE: match_type, + RowSource.SCORE_DATA: (variant.data or {}).get("score_data"), + RowSource.COUNT_DATA: (variant.data or {}).get("count_data"), + } + + for namespace, column_keys in columns.items(): + spec = namespace_spec(namespace) + if spec is None: + continue + + source: Any + # Only the parameterized namespaces carry a distinct datum per namespace. + if spec.source is RowSource.CLINVAR_ENTRY: + source = (clinvar_data_by_ns or {}).get(namespace) + elif spec.source is RowSource.ANNOTATION: + source = (annotations_by_ns or {}).get(namespace) + else: + source = row_sources[spec.source] + + for column_key in column_keys: + resolver = spec.resolver(column_key) + if resolver is None: + raise ValueError(f"unrecognized {namespace} column: {column_key}") + + row[_format_column_key(namespace, column_key, namespaced=namespaced)] = _value_or_na( + resolver(source), na_rep + ) + + return row + + +def variants_to_csv_rows( + variants: Sequence[Variant], + columns: dict[str, list[str]], + mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + annotations_by_ns: Optional[Sequence[Optional[dict[str, Optional[FlatAnnotation]]]]] = None, + match_types: Optional[Sequence[Optional[str]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> Iterable[dict[str, Any]]: + """Format each variant into a dictionary row containing the keys specified in *columns*.""" + n = len(variants) + _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n + _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n + ) + _annotations: Sequence[Optional[dict[str, Optional[FlatAnnotation]]]] = ( + annotations_by_ns if annotations_by_ns is not None else [None] * n + ) + _match_types: Sequence[Optional[str]] = match_types if match_types is not None else [None] * n + return map( + lambda t: variant_to_csv_row( + t[0], + columns, + mapping=t[1], + gnomad_data=t[2], + clinvar_data_by_ns=t[3], + annotations_by_ns=t[4], + match_type=t[5], + namespaced=namespaced, + na_rep=na_rep, + ), + zip(variants, _mappings, _gnomad, _clinvar, _annotations, _match_types), + ) + + +def rows_to_csv(rows: Iterable[dict[str, Any]], columns: list[str]) -> str: + """Serialize *rows* to a CSV string headed by *columns*.""" + stream = io.StringIO() + writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL) + writer.writeheader() + writer.writerows(rows) + return stream.getvalue() + + +def drop_unused_hgvs_columns( + rows_data: Iterable[dict[str, Any]], columns: list[str] +) -> tuple[list[dict[str, Any]], list[str]]: + """Omit the HGVS coordinate columns this score set does not use. + + A protein-only score set never has ``hgvs_nt``; that is a property of the score set, not sparse data. + Limited to the three core HGVS columns on purpose — dropping data-dependent columns elsewhere would + make a download's shape vary with its contents. + + Assumes the "core" namespace is present, which ``plan_csv_columns`` guarantees. + """ + rows_data = list(rows_data) + columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + columns_to_remove = [] + + for col in columns_to_check: + if all(validate_is_null(row[col]) for row in rows_data): + columns_to_remove.append(col) + for row in rows_data: + row.pop(col, None) + + columns = [col for col in columns if col not in columns_to_remove] + return rows_data, columns diff --git a/src/mavedb/lib/csv/deprecated_params.py b/src/mavedb/lib/csv/deprecated_params.py new file mode 100644 index 000000000..3199fce5d --- /dev/null +++ b/src/mavedb/lib/csv/deprecated_params.py @@ -0,0 +1,106 @@ +"""Deprecated query parameters on the score-set CSV endpoints, kept working for backwards compatibility. + +``drop_na_columns`` and ``include_post_mapped_hgvs`` were renamed when the CSV export moved to a namespace +vocabulary. FastAPI ignores unknown query parameters, so a client still sending the old names would have +silently received different output rather than an error, and Galaxy calls these endpoints. + +Requests using a deprecated name get ``Deprecation`` and ``Warning`` response headers, the parameter is +marked deprecated in OpenAPI, and each use is logged so we can see who is left before removal. + +TODO(#XXX): remove this module once clients have migrated. +""" + +import logging +from dataclasses import dataclass, field +from typing import List, Optional + +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.logging.context import save_to_logging_context + +logger = logging.getLogger(__name__) + + +DROP_NA_COLUMNS_DESCRIPTION = ( + "Deprecated: use `drop_unused_hgvs_columns`, which names what it actually does. This parameter only" + " ever dropped the HGVS coordinate columns a score set does not use, never every NA column. It will" + " be removed in a future release; `drop_unused_hgvs_columns` wins if both are given." +) + +INCLUDE_POST_MAPPED_HGVS_DESCRIPTION = ( + "Deprecated: request the `mavedb` namespace instead, e.g. `?namespaces=scores&namespaces=mavedb`." + " Passing true here is equivalent to appending that namespace. It will be removed in a future release." +) + +INCLUDE_CUSTOM_COLUMNS_DESCRIPTION = ( + "Deprecated: request the `scores_custom` namespace instead. Passing true here is equivalent to" + " appending that namespace, whose columns are emitted under the `scores` prefix as before. It will be" + " removed in a future release." +) + + +@dataclass +class ResolvedCsvParams: + """The parameters an endpoint should act on, plus the headers telling the client what it sent.""" + + namespaces: List[str] + drop_unused_hgvs_columns: Optional[bool] + deprecations: dict[str, str] = field(default_factory=dict) + + def _record(self, name: str, replacement: str) -> None: + self.deprecations[name] = replacement + save_to_logging_context({"deprecated_query_parameters": sorted(self.deprecations), "deprecation_marker": True}) + logger.warning( + msg=f"Request used the deprecated query parameter '{name}'; it will be removed in a future" + f" release. Use '{replacement}' instead.", + extra={"deprecated_query_parameter": name, "replacement_query_parameter": replacement}, + ) + + @property + def response_headers(self) -> dict[str, str]: + """Headers announcing the deprecation to the client, or nothing at all for a current request.""" + if not self.deprecations: + return {} + + warnings = "; ".join( + f"{name} is deprecated, use {replacement}" for name, replacement in sorted(self.deprecations.items()) + ) + return { + # RFC 8594. No Sunset header: the removal release is not scheduled. + "Deprecation": "true", + "Warning": f'299 - "{warnings}"', + } + + +def resolve_deprecated_csv_params( + *, + namespaces: Optional[List[str]] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = None, + include_post_mapped_hgvs: Optional[bool] = None, + include_custom_columns: Optional[bool] = None, +) -> ResolvedCsvParams: + """Fold the deprecated spellings into the current ones. + + The current name wins when both are given. The two boolean flags append a namespace rather than + replacing the requested ones, since both were always additive to whatever columns were asked for. + """ + resolved = ResolvedCsvParams( + namespaces=list(namespaces or []), + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + ) + + if drop_unused_hgvs_columns is None and drop_na_columns is not None: + resolved.drop_unused_hgvs_columns = drop_na_columns + resolved._record("drop_na_columns", "drop_unused_hgvs_columns") + + if include_post_mapped_hgvs: + resolved._record("include_post_mapped_hgvs", "namespaces=mavedb") + if CsvNamespace.REFERENCE_HGVS not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.REFERENCE_HGVS) + + if include_custom_columns: + resolved._record("include_custom_columns", "namespaces=scores_custom") + if CsvNamespace.SCORES_CUSTOM not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.SCORES_CUSTOM) + + return resolved diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py new file mode 100644 index 000000000..ea57687e4 --- /dev/null +++ b/src/mavedb/lib/csv/entries.py @@ -0,0 +1,192 @@ +"""Shared pieces for advertising CSV columns: the entry a picker renders, the label builders, and the +two questions about a score set that decide whether a namespace is offerable. + +Each export owns its own discovery function, since what counts as "available" differs: the score-set CSV +asks about one score set, the variant CSV widens across every score set measuring the same allele. +""" + +from dataclasses import dataclass +from typing import Callable, Iterable, Optional, Sequence + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session + +from mavedb.lib.annotation.util import score_calibration_may_be_used_for_annotation +from mavedb.lib.csv.namespaces import ( + CLINVAR_DB_NAME, + STATIC_CSV_NAMESPACE_LABELS, + CsvNamespaceGroup, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_label, + clinvar_namespace_sort_key, +) +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class AvailableCsvNamespaceEntry: + """A namespace a record has data for, labeled and grouped for a picker.""" + + namespace: str + label: str + group: CsvNamespaceGroup + + score_set: Optional[ScoreSet] = None + """Owning score set, set for calibration namespaces only. + + A calibration means nothing against another score set's scores, and the variant CSV widens across + several score sets, so a picker needs this to tell their calibrations apart. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges. Answers only "what + should a dialog open on" . Do not read this as a publish/include policy. + """ + + research_use_only: bool = False + """Whether the data comes from a research-use-only calibration. + + Separate from ``selected_by_default`` so a consumer deciding what may be published can ask directly. + """ + + +def static_namespace_entry(namespace: str) -> AvailableCsvNamespaceEntry: + """Build the labeled entry for a static namespace.""" + label, group = STATIC_CSV_NAMESPACE_LABELS[namespace] + return AvailableCsvNamespaceEntry(namespace=namespace, label=label, group=group) + + +def clinvar_namespace_entries(namespaces: Iterable[str]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for ClinVar release namespaces, newest first, newest selected by default.""" + entries: list[AvailableCsvNamespaceEntry] = [] + # Chronological key, not string order: this sort decides which release opens checked. + for namespace in sorted(set(namespaces), key=clinvar_namespace_sort_key, reverse=True): + label = clinvar_namespace_label(namespace) + if label is not None: + entries.append( + AvailableCsvNamespaceEntry( + namespace=namespace, + label=label, + group=CsvNamespaceGroup.ANNOTATION, + selected_by_default=not entries, # first to survive labelling wins the default + ) + ) + + return entries + + +def visible_calibrations( + calibrations: Iterable[ScoreCalibration], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> list[ScoreCalibration]: + """Drop calibrations the caller may not read. + + A calibration carries its own ``private`` flag, and its READ permission is stricter than its score + set's: a private one is readable only by its owner, by contributors when it is investigator-provided, + or by an admin. Reading the score set is not enough, so every path that names a calibration has to ask + separately. + + Defaults to public-only, and treats an unset ``private`` as private. A caller that forgets to pass a + predicate therefore gets the subset anyone could see rather than everything. + """ + permitted = may_read_calibration or (lambda calibration: calibration.private is False) + return [calibration for calibration in calibrations if permitted(calibration)] + + +def calibration_can_annotate(calibration: ScoreCalibration) -> bool: + """Whether a calibration can support either kind of annotation, and so fill any of its columns. + + False for a calibration with no score ranges, whose every cell would be NA. Research-use-only standing + is excluded from this question — it asks what a calibration *could* say, while who may see it is + ``visible_calibrations``' job. + """ + return any( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type=annotation_type, # type: ignore[arg-type] + allow_research_use_only_calibrations=True, + ) + for annotation_type in ("functional", "pathogenicity") + ) + + +def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for calibrations, named by title so a picker can identify them. + + Research-use-only calibrations (labelled with a prefix) and rangeless ones are offered but excluded + from the default selection. + """ + entries = [] + for calibration in sorted(calibrations, key=lambda c: (str(c.title or ""), str(c.urn or ""))): + if not calibration.urn: + continue + + title = str(calibration.title) if calibration.title else str(calibration.urn) + research_use_only = bool(calibration.research_use_only) + entries.append( + AvailableCsvNamespaceEntry( + namespace=calibration_namespace_for_urn(str(calibration.urn)), + label=f"Research Use Only: {title}" if research_use_only else title, + group=CsvNamespaceGroup.CALIBRATION, + score_set=calibration.score_set, + research_use_only=research_use_only, + selected_by_default=not research_use_only and calibration_can_annotate(calibration), + ) + ) + + return entries + + +def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) -> bool: + """Whether any variant in these score sets has a current mapping. + + Gates the mapping-derived namespaces: any mapping in a score set means the variant CSV + should offer the namespaces, even if the variant in question is unmapped. + """ + if not score_set_ids: + return False + + return ( + db.scalars( + select(MappedVariant.id) + .join(MappedVariant.variant) + .where(and_(Variant.score_set_id.in_(score_set_ids), MappedVariant.current.is_(True))) + .limit(1) + ).first() + is not None + ) + + +def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> list[str]: + """Every ClinVar release namespace these score sets have data for. + + Scoped to the score set, not the measurement, so a variant with no record still gets NA columns — + an omitted column would read as "never consulted". Keyed on score set ids because deriving them + inside the query measured slower. + """ + if not score_set_ids: + return [] + + db_versions = db.scalars( + select(ClinicalControl.db_version) + .join(ClinicalControl.mapped_variants.of_type(MappedVariant)) + .join(MappedVariant.variant) + .where( + and_( + Variant.score_set_id.in_(score_set_ids), + MappedVariant.current.is_(True), + ClinicalControl.db_name == CLINVAR_DB_NAME, + ) + ) + .distinct() + ).all() + + namespaces = [clinvar_namespace_for_db_version(str(version)) for version in db_versions] + return [namespace for namespace in namespaces if namespace is not None] diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py new file mode 100644 index 000000000..9244db15c --- /dev/null +++ b/src/mavedb/lib/csv/fetch.py @@ -0,0 +1,176 @@ +"""Fetching the rows a CSV export renders — a whole score set, or an explicit set of variants. + +Which relationships are eager-loaded follows from the requested namespaces, so a caller cannot forget one +and silently pay for an N+1. +""" + +from dataclasses import dataclass +from typing import Any, Optional, Sequence + +from sqlalchemy import Integer, and_, cast, func, select +from sqlalchemy.orm import Session, aliased, selectinload + +from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME +from mavedb.lib.csv.specs import namespace_spec +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass +class CsvFetchResult: + variants: list[Variant] + mappings: Optional[list[Optional[MappedVariant]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + + +def fetch_variant_csv_data( + db: Session, + namespaced_columns: dict[str, list[str]], + clinvar_namespaces: dict[str, str], + *, + score_set: Optional[ScoreSet] = None, + variant_ids: Optional[Sequence[int]] = None, + mapped_variant_ids: Optional[Sequence[int]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, +) -> CsvFetchResult: + """Fetch variant data from the database for CSV generation. + + Args: + score_set: every variant in a score set, ordered by URN suffix. Mutually exclusive with + *variant_ids*, which returns an explicit set in the order given. Exactly one is required. + mapped_variant_ids: pins which mapping stands for each variant. Required from a caller that has + already chosen one, since re-resolving on ``current`` alone could pick a different row and + emit a variant twice — nothing in the schema stops two mappings claiming to be current. + """ + if (score_set is None) == (variant_ids is None): + raise ValueError("exactly one of score_set or variant_ids must be provided") + + # Driven by the namespaces' own descriptors, so a namespace cannot declare a relationship-backed + # column and then quietly not have it loaded. + specs = [spec for spec in (namespace_spec(ns) for ns in namespaced_columns) if spec is not None] + + need_mappings = any(spec.needs_mappings for spec in specs) + need_gnomad = any(spec.needs_gnomad for spec in specs) + need_score_set = any(spec.needs_score_set for spec in specs) + + variants: list[Variant] = [] + mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None + gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + + select_columns: list[Any] = [Variant] + if need_mappings: + select_columns.append(MappedVariant) + if need_gnomad: + select_columns.append(GnomADVariant) + + query = select(*select_columns) + + if score_set is not None: + query = query.where(Variant.score_set_id == score_set.id).order_by( + cast(func.split_part(Variant.urn, "#", 2), Integer) + ) + else: + query = query.where(Variant.id.in_(variant_ids or [])) + + if need_score_set: + query = query.options( + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + + if need_mappings: + mapping_on_clause = ( + and_(Variant.id == MappedVariant.variant_id, MappedVariant.id.in_(mapped_variant_ids)) + if mapped_variant_ids is not None + else and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)) + ) + query = query.join(MappedVariant, mapping_on_clause, isouter=True) + + # Version predicate belongs in the ON clause: in a WHERE it would drop any variant linked only to + # other-version gnomAD records from the CSV entirely, instead of reporting its frequency as NA. + if need_gnomad: + query = query.join( + MappedVariant.gnomad_variants.of_type(GnomADVariant).and_( + GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION + ), + isouter=True, + ) + + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + result = db.execute(query).all() + + # Postgres does not preserve IN-list order, so restore the caller's ordering. + if variant_ids is not None: + position = {variant_id: index for index, variant_id in enumerate(variant_ids)} + result = sorted(result, key=lambda row: position.get(row[0].id, len(position))) + + for row in result: + variant = row[0] + variants.append(variant) + + if need_mappings and mappings is not None: + mappings.append(row[1]) + + if need_gnomad and gnomad_data_list is not None: + idx = 2 if need_mappings else 1 + gnomad_data_list.append(row[idx]) + + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + if clinvar_namespaces and mappings is not None: + mv_ids = [m.id for m in mappings if m is not None] + + # One query per namespace, since each names a different release; keyed by MappedVariant id and + # projected back onto row order below. + clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + for ns, db_version in clinvar_namespaces.items(): + mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + if mv_ids: + aliased_cc = aliased(ClinicalControl) + cc_query = ( + select( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id, + aliased_cc, + ) + .join( + aliased_cc, + mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, + ) + .where( + and_( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), + aliased_cc.db_name == CLINVAR_DB_NAME, + aliased_cc.db_version == db_version, + ) + ) + ) + + for mv_id, cc in db.execute(cc_query).all(): + mv_to_cc[mv_id] = cc + + clinvar_data_map[ns] = mv_to_cc + + clinvar_per_variant = [ + { + ns: mv_to_cc.get(mapping.id) if mapping is not None and mapping.id is not None else None + for ns, mv_to_cc in clinvar_data_map.items() + } + for mapping in mappings + ] + + return CsvFetchResult( + variants=variants, + mappings=mappings, + gnomad_data=gnomad_data_list, + clinvar_per_variant=clinvar_per_variant, + ) diff --git a/src/mavedb/lib/csv/namespaces.py b/src/mavedb/lib/csv/namespaces.py new file mode 100644 index 000000000..0565c6a8c --- /dev/null +++ b/src/mavedb/lib/csv/namespaces.py @@ -0,0 +1,254 @@ +"""The vocabulary of CSV column namespaces: names, labels, validation. + +Most namespaces are fixed names. Two families are parameterized, since their columns depend on which +record the caller wants: ``clinvar.YYYY_MM`` and ``calibration.``. A parameterized namespace carries +its parameter into the column header, keeping values traceable without a separate provenance column. +""" + +import re +from enum import StrEnum +from typing import Annotated, Optional + +from pydantic import AfterValidator, WithJsonSchema + +from mavedb.lib.validation.urn_re import MAVEDB_CALIBRATION_URN_PATTERN + + +class CsvNamespaceGroup(StrEnum): + """Presentational grouping, so a client can section a namespace picker.""" + + DATA = "data" + ANNOTATION = "annotation" + CALIBRATION = "calibration" + PROVENANCE = "provenance" + + +class CsvNamespace(StrEnum): + """Namespaces whose column sets are fixed and take no parameter. + + The parameterized families cannot be members, so requests are validated against this enum *and* those + patterns. See ``is_valid_csv_namespace``. + """ + + SCORES = "scores" + """The one score column every score set is required to define.""" + + SCORES_CUSTOM = "scores_custom" + """The remaining score columns the investigator uploaded. + + A request token only: its columns are emitted under the ``scores`` prefix, since they are score + columns. Splitting selection from emission is what let this replace the ``include_custom_columns`` + flag without changing a published header. + """ + + COUNTS = "counts" + + # Value frozen as "mavedb", and its columns keep their post_mapped_* names: both are published, and + # the score-set histogram parses mavedb.post_mapped_hgvs_c by name. + REFERENCE_HGVS = "mavedb" + + VEP = "vep" + GNOMAD = "gnomad" + CLINGEN = "clingen" + SCORE_SET = "score_set" + RELATIONSHIP = "relationship" + + +STATIC_CSV_NAMESPACES: tuple[str, ...] = tuple(ns.value for ns in CsvNamespace) +"""The static namespace values, in declaration order, for iteration and documentation.""" + +STATIC_CSV_NAMESPACE_LABELS: dict[str, tuple[str, CsvNamespaceGroup]] = { + CsvNamespace.SCORES: ("Score", CsvNamespaceGroup.DATA), + CsvNamespace.SCORES_CUSTOM: ("Investigator-provided score columns", CsvNamespaceGroup.DATA), + CsvNamespace.COUNTS: ("Counts", CsvNamespaceGroup.DATA), + CsvNamespace.CLINGEN: ("ClinGen allele ID", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.REFERENCE_HGVS: ("Reference-frame HGVS", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.VEP: ("VEP consequence", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.GNOMAD: ("gnomAD allele frequency", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.SCORE_SET: ("Score set and target gene", CsvNamespaceGroup.PROVENANCE), + CsvNamespace.RELATIONSHIP: ("Relationship to the requested variant", CsvNamespaceGroup.PROVENANCE), +} +"""Label and group for each static namespace, so a client need not maintain its own mapping. + +The parameterized families are labeled from their parameter instead; see ``clinvar_namespace_label``. +""" + + +_CLINVAR_MONTH_NAMES = ( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +) + + +CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +"""Pattern for ClinVar namespaces of the form ``"clinvar.YEAR_MONTH"``, e.g. ``clinvar.2024_01``.""" + + +CLINVAR_DB_NAME = "ClinVar" +"""The ``clinical_controls.db_name`` a ``clinvar.*`` namespace selects on.""" + + +def parse_clinvar_namespace(ns: str) -> Optional[str]: + """Parse a ClinVar namespace into the ``db_version`` stored in ``clinical_controls``. + + Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` for January 2024). + The corresponding ``db_version`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). + + Returns ``None`` if *ns* does not match the expected pattern. + """ + m = CLINVAR_NS_PATTERN.match(ns) + if not m: + return None + year, month = m.group(1), m.group(2) + return f"{month}_{year}" + + +def parse_clinvar_db_version(db_version: str) -> Optional[tuple[int, int]]: + """Parse a ClinVar ``"MM_YYYY"`` db_version into ``(year, month)``. + + Returns ``None`` when *db_version* is not in the expected form. The tuple orders chronologically, so + it doubles as a sort key for picking the most recent release. + """ + try: + month, year = db_version.split("_") + return (int(year), int(month)) + except (ValueError, AttributeError): + return None + + +_UNDATED_CLINVAR_SORT_KEY = (-1, -1) +"""Sorts before every real release, so a namespace we cannot date never wins a "newest" comparison.""" + + +def clinvar_namespace_sort_key(ns: str) -> tuple[int, int]: + """Chronological sort key for a ClinVar release namespace. + + Use this for every "which release is newest" decision rather than comparing namespace strings: the + year group is unpadded, so ``"clinvar.999_12" > "clinvar.2025_01"`` lexically. Non-release namespaces + sort before every real one. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return _UNDATED_CLINVAR_SORT_KEY + return (int(match.group(1)), int(match.group(2))) + + +CALIBRATION_NS_PATTERN = re.compile(rf"^calibration\.({MAVEDB_CALIBRATION_URN_PATTERN})$") +"""Pattern for calibration namespaces of the form ``"calibration."``.""" + + +def parse_calibration_namespace(ns: str) -> Optional[str]: + """Parse a calibration namespace into the calibration URN it names. + + Returns ``None`` if *ns* does not match the expected pattern. + """ + match = CALIBRATION_NS_PATTERN.match(ns) + if not match: + return None + return match.group(1) + + +def calibration_namespace_for_urn(urn: str) -> str: + """Build the namespace naming a calibration URN.""" + return f"calibration.{urn}" + + +def clinvar_namespace_for_db_version(db_version: str) -> Optional[str]: + """Build the namespace naming a ClinVar release from its ``"MM_YYYY"`` db_version. + + Returns ``None`` when *db_version* is not in the expected form. + """ + parsed = parse_clinvar_db_version(db_version) + if parsed is None: + return None + year, month = parsed + return f"clinvar.{year}_{month:02d}" + + +def clinvar_namespace_label(ns: str) -> Optional[str]: + """Human-readable label for a ClinVar release namespace, e.g. ``"ClinVar significance (November 2024)"``. + + Returns ``None`` when *ns* is not a ClinVar namespace. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return None + year, month = int(match.group(1)), int(match.group(2)) + return f"ClinVar significance ({_CLINVAR_MONTH_NAMES[month - 1]} {year})" + + +_STATIC_CSV_NAMESPACE_VALUES = frozenset(STATIC_CSV_NAMESPACES) +"""Membership set. ``"scores" in CsvNamespace`` raises TypeError on Python 3.11, so test against this.""" + + +def is_valid_csv_namespace(ns: str) -> bool: + """Whether *ns* is a namespace any CSV endpoint will accept.""" + return ( + ns in _STATIC_CSV_NAMESPACE_VALUES + or CLINVAR_NS_PATTERN.match(ns) is not None + or CALIBRATION_NS_PATTERN.match(ns) is not None + ) + + +CSV_NAMESPACE_ERROR_MESSAGE = ( + "must be one of " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + ', a ClinVar release namespace of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01"),' + ' or a calibration namespace of the form "calibration."' +) + + +def _validated_csv_namespace(ns: str) -> str: + """Pydantic validator backing ``CsvNamespaceStr``.""" + if not is_valid_csv_namespace(ns): + raise ValueError(CSV_NAMESPACE_ERROR_MESSAGE) + return ns + + +CsvNamespaceStr = Annotated[ + str, + AfterValidator(_validated_csv_namespace), + # Hand-declared because no Python type expresses "closed enum OR two open patterns". FastAPI then + # rejects bad values itself, so endpoints need no vocabulary check. + # + # Caveat: this does not reach clients as a *type*. openapi-typescript narrows an enum mixed with + # patterns to plain `string`, so generated clients see `string[]` and cannot check a namespace name at + # compile time. Only own-component schemas (CsvNamespaceGroup) survive as a union. + WithJsonSchema( + { + "type": "string", + "anyOf": [ + {"enum": list(STATIC_CSV_NAMESPACES)}, + {"pattern": CLINVAR_NS_PATTERN.pattern}, + {"pattern": CALIBRATION_NS_PATTERN.pattern}, + ], + } + ), +] +"""The type for a ``namespaces`` query-parameter element on any CSV endpoint. + +Use ``Optional[List[CsvNamespaceStr]]`` and FastAPI handles validation and documentation. +""" + + +CSV_NAMESPACES_PARAM_DESCRIPTION = ( + "One or more groups of columns to include. Naming any group replaces the default set rather than " + "adding to it, so list every group you want. Fixed groups: " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + '. Versioned groups: "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01") for one ClinVar release, and ' + '"calibration." for one score calibration\'s functional and ACMG interpretation. ' + "Several ClinVar and calibration namespaces may be requested at once; each carries its release or " + "URN in the column header. To discover which namespaces are available for a record, query the " + "`csv-namespaces` endpoint." +) +"""Shared OpenAPI description for the ``namespaces`` query parameter on every CSV endpoint.""" diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py new file mode 100644 index 000000000..148cc0bd9 --- /dev/null +++ b/src/mavedb/lib/csv/score_set.py @@ -0,0 +1,124 @@ +"""The score-set CSV export: every variant in one score set, and the columns it can offer.""" + +from typing import Callable, List, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_namespace_entries, + visible_calibrations, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet + + +def get_score_set_variants_as_csv( + db: Session, + score_set: ScoreSet, + namespaces: List[str], + namespaced: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, + drop_unused_hgvs_columns_flag: Optional[bool] = None, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> str: + """Get the variant data from a score set as a CSV string.""" + assert type(score_set.dataset_columns) is dict + + plan = plan_csv_columns(score_set.dataset_columns, namespaces) + + fetched = fetch_variant_csv_data( + db, + plan.namespaced_columns, + plan.clinvar_namespaces, + score_set=score_set, + start=start, + limit=limit, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + rows_data = variants_to_csv_rows( + fetched.variants, + columns=plan.namespaced_columns, + namespaced=namespaced, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows( + db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration + ), + ) + + rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) + + if drop_unused_hgvs_columns_flag: + rows_data, rows_columns = drop_unused_hgvs_columns(rows_data, rows_columns) + + return rows_to_csv(rows_data, rows_columns) + + +def available_score_set_csv_namespaces( + db: Session, + score_set: ScoreSet, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. + + Its own endpoint rather than a field on the score-set response: it costs several queries and is only + needed when a download dialog opens. A namespace absent here is still accepted by the CSV endpoint; + it just produces a column of NA. + """ + dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} + # TODO(#372): non-null id fields + score_set_ids: list[int] = [score_set.id] # type: ignore + + score_columns = [str(column) for column in dataset_columns.get("score_columns", [])] + + entries: list[AvailableCsvNamespaceEntry] = [] + if score_columns: + entries.append(static_namespace_entry(CsvNamespace.SCORES)) + if any(column != REQUIRED_SCORE_COLUMN for column in score_columns): + entries.append(static_namespace_entry(CsvNamespace.SCORES_CUSTOM)) + if dataset_columns.get("count_columns"): + entries.append(static_namespace_entry(CsvNamespace.COUNTS)) + + entries.append(static_namespace_entry(CsvNamespace.SCORE_SET)) # always its own provenance + + if score_sets_have_current_mappings(db, score_set_ids): + entries.extend( + static_namespace_entry(ns) + for ns in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.GNOMAD, CsvNamespace.CLINGEN) + ) + entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids))) + + # Every calibration the score set defines is offered, rangeless ones included. + calibrations = db.scalars( + select(ScoreCalibration) + .options( + selectinload(ScoreCalibration.score_set), + selectinload(ScoreCalibration.functional_classifications), # read by the eligibility check + ) + .where(and_(ScoreCalibration.score_set_id == score_set.id, ScoreCalibration.urn.is_not(None))) + ).all() + entries.extend(calibration_namespace_entries(visible_calibrations(calibrations, may_read_calibration))) + + # `relationship` is absent by design: match_type describes a row's relation to a requested record, + # which only the variant CSV has. + return entries diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py new file mode 100644 index 000000000..b3b671598 --- /dev/null +++ b/src/mavedb/lib/csv/specs.py @@ -0,0 +1,254 @@ +"""What each CSV namespace is: the columns it produces and how each is read off a row.""" + +from dataclasses import dataclass +from enum import StrEnum +from operator import attrgetter +from typing import Callable, Optional + +from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +# One entry per namespace, so adding one is a single edit. This previously took three unrelated changes +# — column plan, row builder, fetch-layer eager loading — with nothing to catch a partial addition. + + +CORE_NAMESPACE = "core" +"""The identity columns every export carries, never namespaced and never opted out of.""" + + +class DatasetColumnSelection(StrEnum): + """Which of a ``dataset_columns`` entry's columns a namespace claims. + + Only the score columns are split, because ``score`` is the one column dataframe validation mandates, + which makes "the required column" and "everything else" well defined. + """ + + ALL = "all" + + REQUIRED_SCORE_ONLY = "required_score_only" + """Just ``score``, without consulting the record: dataframe validation mandates the column, so callers + with no ``dataset_columns`` to hand (the variant CSV) still resolve it.""" + + EXCEPT_REQUIRED_SCORE = "except_required_score" + """Everything else, which only the record can enumerate.""" + + +class RowSource(StrEnum): + """Which per-row datum a namespace's resolvers are called with.""" + + VARIANT = "variant" + MAPPING = "mapping" + GNOMAD = "gnomad" + MATCH_TYPE = "match_type" + SCORE_DATA = "score_data" + COUNT_DATA = "count_data" + + # Parameterized namespaces are keyed by the namespace string, since one row carries a separate datum + # for every requested release or calibration. + CLINVAR_ENTRY = "clinvar_entry" + ANNOTATION = "annotation" + + +@dataclass(frozen=True) +class CsvNamespaceSpec: + """Everything one namespace contributes to an export.""" + + source: RowSource + """Which per-row datum the resolvers are called with.""" + + resolvers: Optional[dict[str, Callable]] = None + """Column key -> how to read it off *source*. None means the columns are not known ahead of time and + are read by key, which is how a score set's own score and count columns work.""" + + dataset_columns_key: Optional[str] = None + """The ``dataset_columns`` entry listing this namespace's columns, for namespaces whose columns come + from the score set rather than from this module.""" + + dataset_columns: DatasetColumnSelection = DatasetColumnSelection.ALL + """Which of that entry's columns this namespace claims.""" + + emit_under: Optional[str] = None + """Prefix these columns are emitted under, when it differs from the namespace's own name. + + Lets a namespace be a request token without becoming a column prefix, so ``scores_custom`` selects + while its columns stay ``scores.*``. None means emit under this namespace's own name. + """ + + needs_mappings: bool = False + """Whether the fetch layer has to load the row's mapping for this namespace to work.""" + needs_gnomad: bool = False + """Whether the fetch layer has to load the row's gnomAD data for this namespace to work.""" + needs_score_set: bool = False + """Whether the fetch layer has to load the row's score set for this namespace to work.""" + + def columns(self, dataset_columns: dict) -> list[str]: + """The column keys this namespace produces for a given score set.""" + if self.resolvers is not None: + return list(self.resolvers.keys()) + if self.dataset_columns_key is None: + return [] + + if self.dataset_columns is DatasetColumnSelection.REQUIRED_SCORE_ONLY: + return [REQUIRED_SCORE_COLUMN] + + available = [str(column) for column in dataset_columns.get(self.dataset_columns_key, [])] + if self.dataset_columns is DatasetColumnSelection.EXCEPT_REQUIRED_SCORE: + return [column for column in available if column != REQUIRED_SCORE_COLUMN] + + return available + + def resolver(self, column_key: str) -> Optional[Callable]: + """How to read *column_key* off a row, or None if it is read by key rather than by resolver.""" + if self.resolvers is not None: + return self.resolvers.get(column_key) + + # Dynamic columns are read straight out of the variant's score or count data by name. + return _optional(lambda data: data.get(column_key)) + + +def _post_mapped_hgvs_g(mapping: Optional[MappedVariant]) -> Optional[str]: + """The genomic HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_g: + return str(mapping.hgvs_g) + fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + return fallback if fallback is not None and is_hgvs_g(fallback) else None + + +def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: + """The protein HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_p: + return str(mapping.hgvs_p) + fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + return fallback if fallback is not None and is_hgvs_p(fallback) else None + + +def _post_mapped_vrs_digest(mapping: Optional[MappedVariant]) -> Optional[str]: + """The digest of the post-mapped VRS object, or None if there is no post-mapped object.""" + if mapping is None or not mapping.post_mapped: + return None + return get_digest_from_post_mapped(mapping.post_mapped) + + +def _target_genes(variant: Variant) -> Optional[str]: + """The target genes of a variant's score set, joined by ``"; "`` or None if there are none.""" + if not variant.score_set: + return None + return "; ".join(str(tg.name) for tg in variant.score_set.target_genes if tg.name) or None + + +def _optional(getter: Callable) -> Callable: + """Lift a resolver over a source that may be absent, which is how a row reports "no data here".""" + return lambda source: getter(source) if source is not None else None + + +_NAMESPACE_SPECS: dict[str, CsvNamespaceSpec] = { + CORE_NAMESPACE: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "accession": attrgetter("urn"), + "hgvs_nt": attrgetter("hgvs_nt"), + "hgvs_splice": attrgetter("hgvs_splice"), + "hgvs_pro": attrgetter("hgvs_pro"), + }, + ), + CsvNamespace.SCORES: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.REQUIRED_SCORE_ONLY, + ), + CsvNamespace.SCORES_CUSTOM: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.EXCEPT_REQUIRED_SCORE, + emit_under=CsvNamespace.SCORES, + ), + CsvNamespace.COUNTS: CsvNamespaceSpec(source=RowSource.COUNT_DATA, dataset_columns_key="count_columns"), + # TODO(#784): under the allele-centric (RT) substrate these move off MappedVariant onto the Allele. + # Both are reached through `mapping`, so each becomes a one-line hop, not a column-contract change. + CsvNamespace.REFERENCE_HGVS: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={ + "post_mapped_hgvs_g": _post_mapped_hgvs_g, + "post_mapped_hgvs_p": _post_mapped_hgvs_p, + "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), + "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), + "post_mapped_vrs_digest": _post_mapped_vrs_digest, + }, + needs_mappings=True, + ), + CsvNamespace.VEP: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"vep_functional_consequence": _optional(lambda mapping: mapping.vep_functional_consequence)}, + needs_mappings=True, + ), + CsvNamespace.GNOMAD: CsvNamespaceSpec( + source=RowSource.GNOMAD, + resolvers={"gnomad_af": _optional(lambda gnomad: gnomad.allele_frequency)}, + needs_mappings=True, + needs_gnomad=True, + ), + CsvNamespace.CLINGEN: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"clingen_allele_id": _optional(lambda mapping: mapping.clingen_allele_id)}, + needs_mappings=True, + ), + CsvNamespace.SCORE_SET: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "score_set_urn": lambda variant: variant.score_set.urn if variant.score_set else None, + "target_gene": _target_genes, + }, + needs_score_set=True, + ), + # TODO(#784): once the variant CSV emits sibling rows, report the shared `projection_group` here + # alongside `match_type`, so a consumer can tell a projected sibling from an independent equivalent. + CsvNamespace.RELATIONSHIP: CsvNamespaceSpec( + source=RowSource.MATCH_TYPE, + # Caller-supplied: only an export that widens beyond one record knows how a row relates to it. + resolvers={"match_type": lambda match_type: match_type}, + ), +} + + +_CLINVAR_SPEC = CsvNamespaceSpec( + source=RowSource.CLINVAR_ENTRY, + resolvers={ + "clinical_significance": _optional(attrgetter("clinical_significance")), + "clinical_review_status": _optional(attrgetter("clinical_review_status")), + }, + needs_mappings=True, +) + +_CALIBRATION_SPEC = CsvNamespaceSpec( + source=RowSource.ANNOTATION, + # The calibration's URN is carried in the column header, so it is not repeated as a column. + resolvers={ + "title": _optional(attrgetter("calibration_title")), + "research_use_only": _optional(attrgetter("research_use_only")), + "functional_classification": _optional(attrgetter("functional_classification")), + "acmg_criterion": _optional(attrgetter("acmg_criterion")), + "acmg_evidence_strength": _optional(attrgetter("acmg_evidence_strength")), + "acmg_evidence_outcome_code": _optional(attrgetter("acmg_evidence_outcome_code")), + "pathogenicity_classification": _optional(attrgetter("pathogenicity_classification")), + }, + needs_mappings=True, + needs_score_set=True, +) + + +def namespace_spec(namespace: str) -> Optional[CsvNamespaceSpec]: + """The descriptor for *namespace*, or None if it names nothing this module can produce.""" + if namespace in _NAMESPACE_SPECS: + return _NAMESPACE_SPECS[namespace] + if CLINVAR_NS_PATTERN.match(namespace): + return _CLINVAR_SPEC + if CALIBRATION_NS_PATTERN.match(namespace): + return _CALIBRATION_SPEC + return None diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py new file mode 100644 index 000000000..66cccdfa0 --- /dev/null +++ b/src/mavedb/lib/csv/variant.py @@ -0,0 +1,388 @@ +"""Clinically-oriented, variant-level CSV export. + +Serves the same interpretation as the variant-level VA-Spec JSON download, but flat: a manuscript +reviewer found the ACMG evidence codes clinically inaccessible when buried in nested evidence lines. + +Column layout, fetching, NA handling, and serialization come from the shared engine in this package. +Only the variant-scoped parts live here: finding measurements that share a ClinGen allele, and choosing +default calibration and ClinVar namespaces. +""" + +import logging +from typing import Any, Callable, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_can_annotate, + calibration_namespace_entries, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, + visible_calibrations, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import ( + CsvNamespace, + calibration_namespace_for_urn, + clinvar_namespace_sort_key, +) +from mavedb.lib.mave.utils import NA_VALUE +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + +logger = logging.getLogger(__name__) + + +ALWAYS_AVAILABLE_NAMESPACES: list[str] = [ + CsvNamespace.SCORES, + CsvNamespace.SCORE_SET, + CsvNamespace.RELATIONSHIP, +] +"""Namespaces every measurement can fill: its score, its score set, and its relation to the request.""" + +MAPPING_DERIVED_NAMESPACES: list[str] = [ + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, +] +"""Namespaces read from a mapped variant, offered whenever the score set has any mapping. + +Scoped to the score set, not the measurement: omitting a column would say "never looked" where the truth +is "looked, found nothing". +""" + +BASE_VARIANT_CSV_NAMESPACES: list[str] = ALWAYS_AVAILABLE_NAMESPACES + MAPPING_DERIVED_NAMESPACES +"""The fixed namespaces a mapped variant's CSV includes. + +``scores_custom`` and ``counts`` are excluded: they vary across score sets, and this export puts one row +per score set, so their columns would be mostly NA. +""" + +EXACT_MATCH_TYPE = "exact" +"""Measurements sharing the requested variant's ClinGen allele ID — currently the only relationship emitted. + +TODO(#784): widen ``_equivalent_measurements`` to nucleotide/amino-acid equivalence once #791 lands +``equivalent_nt``/``equivalent_aa``; ``relationship.match_type`` then takes more than this one value. +See https://github.com/VariantEffect/mavedb-api/issues/784 +""" + + +def _equivalent_measurements( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, +) -> Optional[list[tuple[int, int, int]]]: + """Resolve a variant URN to the measurements the CSV should report. + + Returns ``[(variant_id, mapped_variant_id, score_set_id), ...]``, requested variant first, then every + other current measurement of the same ClinGen allele ordered by score set and variant URN so repeated + downloads are byte-identical. At most one entry per variant, which is what lets the fetch layer + restore this order from variant ids alone. + + Args: + may_read_score_set: when given, drops measurements from score sets the caller may not read. + Applied only to score sets reached by the widening; the caller's own permission check on the + requested variant is not repeated here. + + Returns: + None when the variant has no current mapping, since there is then no allele to expand by. + """ + # Nothing in the schema enforces one current mapping per variant, and every column below follows from + # this pick, so order explicitly: an unordered LIMIT 1 would let one URN download differently twice. + requested = db.scalars( + select(MappedVariant) + .join(MappedVariant.variant) + .where(and_(Variant.urn == variant_urn, MappedVariant.current.is_(True))) + .order_by(MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + .limit(1) + ).one_or_none() + + if requested is None: + return None + + # TODO(#372): non-null id fields + if not requested.clingen_allele_id: + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] # type: ignore + + # TODO(#784): under the allele-centric (RT) substrate, replace this string match with the + # projection-aware resolver — measurements attach to an Allele and c/g pairs relate through + # `projection_group`. Only this query changes; callers consume (variant_id, mapped_variant_id) pairs. + equivalents = db.execute( + select(MappedVariant.variant_id, MappedVariant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) + .join(MappedVariant.variant) + .join(Variant.score_set) + .where( + and_( + MappedVariant.clingen_allele_id == requested.clingen_allele_id, + MappedVariant.current.is_(True), + # By variant, not by mapping: the anchor already represents this variant, and a second + # current mapping on it is the same measurement again, not an equivalent one. + MappedVariant.variant_id != requested.variant_id, + ) + ) + .order_by(MappedVariant.variant_id, MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + ).all() + + # One row per variant, picked as the anchor was. A duplicate would defeat the downstream row-order + # restoration, which keys on variant id. TODO(#784): moot once a measurement points at one Allele. + deduplicated: dict[int, Any] = {} + for row in equivalents: + deduplicated.setdefault(row[0], row) + equivalents = list(deduplicated.values()) + + if may_read_score_set is not None and equivalents: + candidate_score_set_ids = {row[2] for row in equivalents} + readable_score_set_ids = { + score_set.id + for score_set in db.scalars(select(ScoreSet).where(ScoreSet.id.in_(candidate_score_set_ids))).all() + if may_read_score_set(score_set) + } + equivalents = [row for row in equivalents if row[2] in readable_score_set_ids] + + equivalents = sorted( + equivalents, + key=lambda row: (score_set_urn_sort_key(row[3]), variant_urn_sort_key(row[4])), + ) + + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] + [ + (row[0], row[1], row[2]) for row in equivalents + ] + + +def _unmapped_variant_namespaces(db: Session, score_set_id: int) -> list[str]: + """The namespaces to offer for a variant that exists but has no current mapping. + + Always score, score set, and relationship; plus the mapping-derived groups when the score set has been + mapped at all, where NA is the honest value for a variant the mapper has not reached. + """ + if score_sets_have_current_mappings(db, [score_set_id]): + return list(BASE_VARIANT_CSV_NAMESPACES) + return list(ALWAYS_AVAILABLE_NAMESPACES) + + +def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional[str]: + """The ClinVar namespace for the most recent release covering these measurements' score sets. + + One release for the whole file rather than each variant's own latest: that keeps the release in the + column header where it is citable, and avoids mixing calls from different releases in one column. + """ + namespaces = clinvar_release_namespaces(db, score_set_ids) + if not namespaces: + return None + + return max(namespaces, key=clinvar_namespace_sort_key) + + +def _annotatable_calibration_namespaces( + db: Session, + score_set_ids: list[int], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> dict[str, ScoreCalibration]: + """Map calibration namespace to calibration, for every calibration eligible to annotate these variants. + + A measurement is only interpretable under its own score set's calibrations, so widening across score + sets widens this set too; a row shows NA under any calibration that does not apply to it. + Research-use-only calibrations are included here but excluded from the default selection. + """ + # Keyed on score sets rather than joined through their variants: `ScoreSet.variants` multiplies the + # join by every variant before DISTINCT collapses it again, for the same result. + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.score_set_id.in_(score_set_ids)) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ), + # Each entry reports the score set it belongs to, so a picker can tell one score set's + # calibrations from another's when the export widens across several. + selectinload(ScoreCalibration.score_set), + ) + ).all() + + namespaces: dict[str, ScoreCalibration] = {} + for calibration in visible_calibrations(calibrations, may_read_calibration): + if not calibration.urn: + continue + + # Dropped outright, where the score-set export merely leaves it unchecked: a variant's + # calibrations are scoped to what interprets *this* allele. + if not calibration_can_annotate(calibration): + continue + + namespaces[calibration_namespace_for_urn(str(calibration.urn))] = calibration + + return namespaces + + +def available_variant_csv_namespaces( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. + + The fixed namespaces, one ``calibration.`` per eligible calibration across the variant's + equivalent measurements, and one ``clinvar.YYYY_MM`` per release covering them. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + variant = db.scalars(select(Variant).where(Variant.urn == variant_urn).limit(1)).first() + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + # No mapping on this variant, but its score set may still be mapped, in which case the + # mapping-derived columns are owed with NA rather than omitted. + # TODO(#372): non-null id fields + return [static_namespace_entry(ns) for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id))] # type: ignore + + base_entries = [static_namespace_entry(ns) for ns in BASE_VARIANT_CSV_NAMESPACES] + + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + return ( + base_entries + + calibration_namespace_entries( + _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration).values() + ) + + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) + ) + + +def get_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the clinical CSV for a variant and its equivalent measurements. + + One row per measurement: the requested variant first, then every other current measurement of the + same ClinGen allele across score sets. + + Args: + namespaces: columns to include, in the same vocabulary as the score-set CSV. When omitted, + defaults to the fixed namespaces plus every eligible calibration and the latest ClinVar + release covering these measurements. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep) + + variant_ids = [variant_id for variant_id, _, _ in measurements] + mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration) + + if namespaces is None: + clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) + # Research-use-only calibrations are offerable but never defaulted to. This export is framed clinically. + default_calibrations = sorted( + namespace for namespace, calibration in calibrations_by_ns.items() if not calibration.research_use_only + ) + resolved_namespaces = ( + BASE_VARIANT_CSV_NAMESPACES + default_calibrations + ([clinvar_namespace] if clinvar_namespace else []) + ) + else: + resolved_namespaces = list(namespaces) + + # Only the required score column is taken, so the score set's own dataset columns are irrelevant. + plan = plan_csv_columns(dataset_columns={}, namespaces=resolved_namespaces) + columns = plan.namespaced_columns + + fetched = fetch_variant_csv_data( + db, + columns, + plan.clinvar_namespaces, + variant_ids=variant_ids, + mapped_variant_ids=mapped_variant_ids, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + + rows = variants_to_csv_rows( + fetched.variants, + columns, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows( + db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration + ), + match_types=[EXACT_MATCH_TYPE] * len(fetched.variants), + na_rep=na_rep, + namespaced=True, + ) + + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) + + +def _unmapped_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the single-row CSV for a variant that exists but has no current mapping. + + Without a mapping there are no coordinates, no external annotations, no allele to find equivalents by, + and nothing for the annotation layer to flatten. Identity, score, and provenance still resolve, so the + download succeeds rather than 404ing on a variant that genuinely exists. + """ + variant = db.scalars( + select(Variant) + .where(Variant.urn == variant_urn) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + ).one_or_none() + + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + plan = plan_csv_columns( + dataset_columns={}, + namespaces=( + # TODO(#372): non-null id fields + list(namespaces) if namespaces is not None else _unmapped_variant_namespaces(db, int(variant.score_set_id)) # type: ignore + ), + ) + columns = plan.namespaced_columns + + rows: list[dict[str, Any]] = list( + variants_to_csv_rows( + [variant], + columns, + match_types=[EXACT_MATCH_TYPE], + na_rep=na_rep, + namespaced=True, + ) + ) + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) diff --git a/src/mavedb/lib/mave/utils.py b/src/mavedb/lib/mave/utils.py index 214532f37..f446150dc 100644 --- a/src/mavedb/lib/mave/utils.py +++ b/src/mavedb/lib/mave/utils.py @@ -3,6 +3,11 @@ import pandas as pd NA_VALUE = "NA" +"""The MAVE convention for a missing value: written by the CSV exports, recognised by the CSV ingest. + +Shared vocabulary rather than an export detail, which is why it stays here while the export-side null +predicate lives with the exporter in ``lib/csv/columns.py``. +""" NULL_VALUES = ("", "na", "nan", "nil", "none", "null", "n/a", "undefined", NA_VALUE) @@ -31,12 +36,3 @@ def is_csv_null(value): if value == 0: return value return not value or NULL_VALUES_RE.fullmatch(str(value).strip().lower()) - - -_CSV_OUTPUT_NULL_RE = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) - - -def is_csv_output_null(value): - """Return True if a value should be replaced with the NA sentinel in CSV output.""" - value = str(value).strip().lower() - return _CSV_OUTPUT_NULL_RE.fullmatch(value) or not value diff --git a/src/mavedb/lib/score_set_csv.py b/src/mavedb/lib/score_set_csv.py deleted file mode 100644 index b960833e7..000000000 --- a/src/mavedb/lib/score_set_csv.py +++ /dev/null @@ -1,484 +0,0 @@ -import csv -import io -from dataclasses import dataclass -from operator import attrgetter -from typing import Any, Callable, Iterable, List, Optional, Sequence - -from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Session, aliased - -from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN -from mavedb.lib.clinvar.utils import parse_clinvar_namespace -from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN -from mavedb.lib.mave.utils import NA_VALUE, is_csv_output_null -from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant - - -@dataclass(frozen=True) -class CsvColumnPlan: - namespaced_columns: dict[str, list[str]] - clinvar_namespaces: dict[str, str] - - -@dataclass -class CsvFetchResult: - variants: list[Variant] - mappings: Optional[list[Optional[MappedVariant]]] - gnomad_data: Optional[list[Optional[GnomADVariant]]] - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] - - -# --------------------------------------------------------------------------- -# Column-key resolvers for fixed-column namespaces -# --------------------------------------------------------------------------- - -_CORE_RESOLVERS: dict[str, Callable] = { - "hgvs_nt": attrgetter("hgvs_nt"), - "hgvs_pro": attrgetter("hgvs_pro"), - "hgvs_splice": attrgetter("hgvs_splice"), - "accession": attrgetter("urn"), -} - -_VEP_RESOLVERS: dict[str, Callable] = { - "vep_functional_consequence": lambda mapping: mapping.vep_functional_consequence if mapping else None, -} - -_GNOMAD_RESOLVERS: dict[str, Callable] = { - "gnomad_af": lambda gnomad_data: gnomad_data.allele_frequency if gnomad_data else None, -} - -_CLINGEN_RESOLVERS: dict[str, Callable] = { - "clingen_allele_id": lambda mapping: mapping.clingen_allele_id if mapping else None, -} - -_CLINVAR_RESOLVERS: dict[str, Callable] = { - "clinical_significance": attrgetter("clinical_significance"), - "clinical_review_status": attrgetter("clinical_review_status"), -} - - -def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: - """Return the string representation of *value*, or *na_rep* if the value is None.""" - if is_csv_output_null(value): - return na_rep - return str(value) - - -def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: - """Shared key-formatting logic used by both header assembly and row assembly.""" - # ClinVar columns are always namespaced to differentiate versions, even if the user has requested un-namespaced output. - if CLINVAR_NS_PATTERN.match(namespace): - return f"{namespace}.{column_key}" - - # The "core" namespace is always un-namespaced, even if the user has requested namespaced output. - if namespace == "core": - return column_key - - # All other namespaces are namespaced if the user has requested namespaced output, and un-namespaced otherwise. - if namespaced: - return f"{namespace}.{column_key}" - - return column_key - - -def _custom_columns(dataset_columns: dict, col_name: str) -> list[str]: - return [col for col in [str(x) for x in list(dataset_columns.get(col_name, []))]] - - -# --------------------------------------------------------------------------- -# Pure functions -# --------------------------------------------------------------------------- - - -def plan_csv_columns( - dataset_columns: dict, - namespaces: list[str], - *, - include_custom_columns: bool = True, - include_post_mapped_hgvs: bool = False, -) -> CsvColumnPlan: - """Build the namespaced column map and ClinVar namespace mapping.""" - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if "scores" in namespaced_score_set_columns: - if include_custom_columns: - # the required score column is transitively included via the _custom_columns function. - namespaced_score_set_columns["scores"] = _custom_columns(dataset_columns, "score_columns") - else: - namespaced_score_set_columns["scores"] = [REQUIRED_SCORE_COLUMN] - if "counts" in namespaced_score_set_columns: - if include_custom_columns: - namespaced_score_set_columns["counts"] = _custom_columns(dataset_columns, "count_columns") - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - clinvar_namespaces: dict[str, str] = {} - for ns in namespaces: - db_version = parse_clinvar_namespace(ns) - if db_version is not None: - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - return CsvColumnPlan( - namespaced_columns=namespaced_score_set_columns, - clinvar_namespaces=clinvar_namespaces, - ) - - -def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: - """Build the flat column-header list from the namespace dict.""" - return [ - _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols - ] - - -# --------------------------------------------------------------------------- -# Row assembly -# --------------------------------------------------------------------------- - - -def variant_to_csv_row( - variant: Variant, - columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, - gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, - namespaced: bool = False, - na_rep=NA_VALUE, -) -> dict[str, Any]: - """Format a variant into a dict containing the keys specified in *columns*.""" - row: dict[str, Any] = {} - - for column_key in columns.get("core", []): - resolver = _CORE_RESOLVERS.get(column_key) - if resolver is None: - raise ValueError(f"unrecognized core column: {column_key}") - - value = str(resolver(variant)) - row[column_key] = _value_or_na(value, na_rep) - - for column_key in columns.get("mavedb", []): - if column_key == "post_mapped_hgvs_g": - value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_p": - value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_c": - value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep - elif column_key == "post_mapped_hgvs_at_assay_level": - value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep - elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep - else: - raise ValueError(f"unrecognized mavedb column: {column_key}") - - row[_format_column_key("mavedb", column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - for ns in ("vep", "gnomad", "clingen"): - resolvers = {"vep": _VEP_RESOLVERS, "gnomad": _GNOMAD_RESOLVERS, "clingen": _CLINGEN_RESOLVERS}[ns] - source = {"vep": mapping, "gnomad": gnomad_data, "clingen": mapping}[ns] - for column_key in columns.get(ns, []): - resolver = resolvers.get(column_key) - if resolver is None: - raise ValueError(f"unrecognized {ns} column: {column_key}") - value = resolver(source) - row[_format_column_key(ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - for data_ns in ("scores", "counts"): - data_key = f"{data_ns[:-1]}_data" - parent = variant.data.get(data_key) if variant.data else None - for column_key in columns.get(data_ns, []): - value = str(parent.get(column_key)) if parent else na_rep - row[_format_column_key(data_ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - for namespace_key, namespace_cols in columns.items(): - if not CLINVAR_NS_PATTERN.match(namespace_key): - continue - clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) - for column_key in namespace_cols: - resolver = _CLINVAR_RESOLVERS.get(column_key) - if resolver is None: - raise ValueError(f"unrecognized clinvar column: {column_key}") - value = str(resolver(clinvar_entry)) if clinvar_entry else na_rep - row[_format_column_key(namespace_key, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - return row - - -def variants_to_csv_rows( - variants: Sequence[Variant], - columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, - gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, - namespaced: bool = False, - na_rep=NA_VALUE, -) -> Iterable[dict[str, Any]]: - """Format each variant into a dictionary row containing the keys specified in *columns*.""" - n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n - _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( - clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n - ) - return map( - lambda t: variant_to_csv_row( - t[0], - columns, - mapping=t[1], - gnomad_data=t[2], - clinvar_data_by_ns=t[3], - namespaced=namespaced, - na_rep=na_rep, - ), - zip(variants, _mappings, _gnomad, _clinvar), - ) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def drop_na_columns_from_csv_file_rows( - rows_data: Iterable[dict[str, Any]], columns: list[str] -) -> tuple[list[dict[str, Any]], list[str]]: - """Process rows_data for downloadable CSV by removing empty columns.""" - rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - columns_to_remove = [] - - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): - columns_to_remove.append(col) - for row in rows_data: - row.pop(col, None) - - columns = [col for col in columns if col not in columns_to_remove] - return rows_data, columns - - -# --------------------------------------------------------------------------- -# DB-bound fetching -# --------------------------------------------------------------------------- - - -def fetch_variant_csv_data( - db: Session, - score_set: ScoreSet, - namespaced_columns: dict[str, list[str]], - clinvar_namespaces: dict[str, str], - *, - include_post_mapped_hgvs: bool = False, - start: Optional[int] = None, - limit: Optional[int] = None, -) -> CsvFetchResult: - """Fetch variant data from the database for CSV generation.""" - namespaces = list(namespaced_columns.keys()) - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) - ) - need_gnomad = "gnomad" in namespaces - - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) - ) - - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data_list is not None: - idx = 2 if need_mappings else 1 - gnomad_data_list.append(row[idx]) - - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - - return CsvFetchResult( - variants=variants, - mappings=mappings, - gnomad_data=gnomad_data_list, - clinvar_per_variant=clinvar_per_variant, - ) - - -# --------------------------------------------------------------------------- -# Public composer -# --------------------------------------------------------------------------- - - -def get_score_set_variants_as_csv( - db: Session, - score_set: ScoreSet, - namespaces: List[str], - namespaced: bool = False, - start: Optional[int] = None, - limit: Optional[int] = None, - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = True, - include_post_mapped_hgvs: Optional[bool] = False, -) -> str: - """Get the variant data from a score set as a CSV string.""" - assert type(score_set.dataset_columns) is dict - - plan = plan_csv_columns( - score_set.dataset_columns, - namespaces, - include_custom_columns=bool(include_custom_columns), - include_post_mapped_hgvs=bool(include_post_mapped_hgvs), - ) - - fetched = fetch_variant_csv_data( - db, - score_set, - plan.namespaced_columns, - plan.clinvar_namespaces, - include_post_mapped_hgvs=bool(include_post_mapped_hgvs), - start=start, - limit=limit, - ) - - rows_data = variants_to_csv_rows( - fetched.variants, - columns=plan.namespaced_columns, - namespaced=namespaced, - mappings=fetched.mappings, - gnomad_data=fetched.gnomad_data, - clinvar_data_by_ns=fetched.clinvar_per_variant, - ) - - rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) - - if drop_na_columns: - rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) - - stream = io.StringIO() - writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) - writer.writeheader() - writer.writerows(rows_data) - return stream.getvalue() diff --git a/src/mavedb/lib/urns.py b/src/mavedb/lib/urns.py index 55a59e707..46be37a10 100644 --- a/src/mavedb/lib/urns.py +++ b/src/mavedb/lib/urns.py @@ -1,11 +1,16 @@ import logging import re import string +from typing import Optional from uuid import uuid4 from sqlalchemy import func from sqlalchemy.orm import Session +from mavedb.lib.validation.urn_re import ( + MAVEDB_EXPERIMENT_SET_URN_DIGITS, + MAVEDB_URN_NAMESPACE, +) from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.score_set import ScoreSet @@ -175,3 +180,55 @@ def generate_job_run_urn(): :return: A new job run URN """ return f"urn:mavedb:job-{uuid4()}" + + +# MaveDB URNs do not sort into assignment order as strings: score-set and variant suffixes are unpadded +# (`-a-10` < `-a-2`, `#10` < `#2`) and experiment suffixes run a..z then aa..az (`aa` < `b`). Only the +# experiment-set digits are padded, which is why a lexical sort looks right until double digits. The keys +# below are the read side of the rule `generate_experiment_urn` already applies when assigning. + + +_SCORE_SET_URN_PARTS_RE = re.compile( + rf"^(?Purn:{MAVEDB_URN_NAMESPACE}:\d{{{MAVEDB_EXPERIMENT_SET_URN_DIGITS}}})" + r"-(?P[a-z]+|0)" + r"-(?P[1-9]\d*)$" +) + +_VARIANT_URN_PARTS_RE = re.compile(r"^(?P.+)#(?P[1-9]\d*)$") + +_UNPARSED = 1 +"""Leading element for an undecomposable URN, so it sorts after every well-formed one. + +Unpublished records carry ``tmp:`` URNs; they still order stably, by the URN itself. Returning a key +rather than raising keeps a temporary URN from turning into a query error. +""" + +_PARSED = 0 + + +def score_set_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int, str, int]: + """Sort key ordering score set URNs the way their parts were assigned.""" + if not urn: + return (_UNPARSED, "", 0, "", 0) + + match = _SCORE_SET_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0, "", 0) + + experiment = match["experiment"] + return (_PARSED, match["experiment_set"], len(experiment), experiment, int(match["score_set"])) + + +def variant_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int]: + """Sort key ordering variant URNs by score set, then by numeric suffix. + + ``...#10`` is the tenth variant of a score set, not something between ``#1`` and ``#2``. + """ + if not urn: + return (_UNPARSED, "", 0) + + match = _VARIANT_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0) + + return (_PARSED, match["score_set"], int(match["number"])) diff --git a/src/mavedb/lib/validation/urn_re.py b/src/mavedb/lib/validation/urn_re.py index 82feb19a2..dddc9d142 100644 --- a/src/mavedb/lib/validation/urn_re.py +++ b/src/mavedb/lib/validation/urn_re.py @@ -32,6 +32,10 @@ MAVEDB_COLLECTION_URN_PATTERN = r"urn:mavedb:collection-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" MAVEDB_COLLECTION_URN_RE = re.compile(MAVEDB_COLLECTION_URN_PATTERN) +# Score calibration URN +MAVEDB_CALIBRATION_URN_PATTERN = r"urn:mavedb:calibration-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +MAVEDB_CALIBRATION_URN_RE = re.compile(MAVEDB_CALIBRATION_URN_PATTERN) + # Any URN MAVEDB_ANY_URN_PATTERN = "|".join( [ diff --git a/src/mavedb/view_models/csv_namespace.py b/src/mavedb/view_models/csv_namespace.py new file mode 100644 index 000000000..e4048cf2e --- /dev/null +++ b/src/mavedb/view_models/csv_namespace.py @@ -0,0 +1,43 @@ +from typing import Optional + +from mavedb.lib.csv.namespaces import CsvNamespaceGroup +from mavedb.view_models import record_type_validator, set_record_type +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_set import ShorterScoreSet + + +class AvailableCsvNamespace(BaseModel): + """One CSV column namespace a record has data for, ready to be offered as a choice. + + Labels are served rather than derived client-side: only the server knows a calibration's title or a + ClinVar release date. + """ + + record_type: str = None # type: ignore + + namespace: str + """The value to pass back in the ``namespaces`` query parameter.""" + + label: str + """Human-readable name for a picker.""" + + group: CsvNamespaceGroup + """Which section of a picker this belongs in.""" + + score_set: Optional[ShorterScoreSet] = None + """The score set a calibration namespace belongs to; None for namespaces that apply to any. + + A picker should group calibrations by this when a response spans more than one score set. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges: both are offered, but + neither should be swept into a download unasked. + """ + + _record_type_factory = record_type_validator()(set_record_type) + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/score_set.py b/src/mavedb/view_models/score_set.py index 84c445eeb..ce7a0730a 100644 --- a/src/mavedb/view_models/score_set.py +++ b/src/mavedb/view_models/score_set.py @@ -244,16 +244,12 @@ def as_form(cls, **kwargs: Any) -> "ScoreSetUpdateAllOptional": # Define which fields need special JSON parsing json_fields = { "contributors": lambda data: [ContributorCreate.model_validate(c) for c in data] if data else None, - "primary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(p) for p in data - ] - if data - else None, - "secondary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(s) for s in data - ] - if data - else None, + "primary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(p) for p in data] if data else None + ), + "secondary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(s) for s in data] if data else None + ), "doi_identifiers": lambda data: [DoiIdentifierCreate.model_validate(d) for d in data] if data else None, "target_genes": lambda data: [TargetGeneCreate.model_validate(t) for t in data] if data else None, "extra_metadata": lambda data: data, @@ -326,7 +322,10 @@ def generate_primary_and_secondary_publications(cls, data: Any): class ShorterScoreSet(BaseModel): + """A score set's identity: enough to name it in a UI without rooting the display on its URN.""" + urn: str + title: str record_type: str = None # type: ignore _record_type_factory = record_type_validator()(set_record_type) diff --git a/tests/lib/annotation/test_flatten.py b/tests/lib/annotation/test_flatten.py new file mode 100644 index 000000000..21ecfa173 --- /dev/null +++ b/tests/lib/annotation/test_flatten.py @@ -0,0 +1,160 @@ +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +from tests.helpers.mocks.factories import ( + create_mock_acmg_classification, + create_mock_functional_classification, + create_mock_mapped_variant, + create_mock_score_calibration, + create_mock_score_set, +) + + +def _calibration_and_variant(functional_classifications, **calibration_kwargs): + """Build a mock mapped variant whose score set carries a single calibration with *functional_classifications*.""" + score_set = create_mock_score_set() + calibration = create_mock_score_calibration( + functional_classifications=functional_classifications, + score_set=score_set, + **calibration_kwargs, + ) + score_set.score_calibrations = [calibration] + return calibration, create_mock_mapped_variant(score_set=score_set) + + +def _pathogenicity_calibration_and_variant( + criterion=ACMGCriterion.PS3, + evidence_strength=StrengthOfEvidenceProvided.STRONG, + variant_in_range=True, + **calibration_kwargs, +): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.abnormal, + label="Abnormal Range", + range_values=[0.7, 1.0], + acmg_classification=create_mock_acmg_classification( + criterion=criterion, + evidence_strength=evidence_strength, + ), + variant_in_range=variant_in_range, + ) + return _calibration_and_variant([classification], **calibration_kwargs) + + +class TestFlattenAnnotation: + """Tests that flatten_annotation projects a calibration's interpretation onto scalar fields.""" + + def test_pathogenicity_calibration_populates_all_fields(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + urn="urn:mavedb:calibration:1", title="Clinical Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "abnormal" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength == "STRONG" + assert annotation.acmg_evidence_outcome_code == "PS3" + assert annotation.pathogenicity_classification == "PATHOGENIC" + assert annotation.calibration_urn == "urn:mavedb:calibration:1" + assert annotation.calibration_title == "Clinical Calibration" + + def test_benign_criterion_yields_benign_classification(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(criterion=ACMGCriterion.BS3) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_criterion == "BS3" + assert annotation.pathogenicity_classification == "BENIGN" + + def test_functional_only_calibration_omits_acmg_fields(self): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.normal, + label="Normal Range", + range_values=[-1.0, 0.3], + ) + calibration, mapped_variant = _calibration_and_variant( + [classification], urn="urn:mavedb:calibration:2", title="Functional Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "normal" + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + assert annotation.calibration_urn == "urn:mavedb:calibration:2" + assert annotation.calibration_title == "Functional Calibration" + + def test_no_calibration_yields_empty_annotation(self): + mapped_variant = create_mock_mapped_variant() + + assert flatten_annotation(mapped_variant, None) == FlatAnnotation() + + def test_calibration_without_ranges_keeps_its_identity(self): + """Which calibration was consulted is known even when it can classify nothing. + + Reporting it is what distinguishes a calibration that defines no ranges from no calibration at + all, and the public dump carries the former. + """ + calibration, mapped_variant = _calibration_and_variant( + [], urn="urn:mavedb:calibration:3", title="Baseline Only" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.calibration_urn == "urn:mavedb:calibration:3" + assert annotation.calibration_title == "Baseline Only" + assert annotation.research_use_only is False + assert annotation.functional_classification is None + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + + def test_variant_outside_all_ranges_is_uncertain(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(variant_in_range=False) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "indeterminate" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code == "PS3_not_met" + assert annotation.pathogenicity_classification == "UNCERTAIN_SIGNIFICANCE" + + @pytest.mark.parametrize( + "criterion,evidence_strength,expected_code", + [ + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.STRONG, "PS3"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.VERY_STRONG, "PS3_very_strong"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.MODERATE, "PS3_moderate"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.SUPPORTING, "PS3_supporting"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.STRONG, "BS3"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.SUPPORTING, "BS3_supporting"), + ], + ) + def test_evidence_outcome_code(self, criterion, evidence_strength, expected_code): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + criterion=criterion, evidence_strength=evidence_strength + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_outcome_code == expected_code + + def test_moderate_plus_is_preserved(self): + """The VA-Spec annotations must collapse M+ to moderate; a CSV has no such obligation.""" + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + evidence_strength=StrengthOfEvidenceProvided.MODERATE_PLUS + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_strength == "MODERATE_PLUS" + assert annotation.acmg_evidence_outcome_code == "PS3_moderate_plus" + assert annotation.pathogenicity_classification == "PATHOGENIC" diff --git a/tests/lib/clinvar/test_utils.py b/tests/lib/clinvar/test_utils.py index 257402d38..082c55586 100644 --- a/tests/lib/clinvar/test_utils.py +++ b/tests/lib/clinvar/test_utils.py @@ -9,7 +9,6 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import ( fetch_clinvar_variant_data, - parse_clinvar_namespace, validate_clinvar_variant_summary_date, ) @@ -46,25 +45,6 @@ def _make_gzipped_tsv(text: str) -> bytes: ) -@pytest.mark.unit -@pytest.mark.parametrize( - "ns, expected", - [ - ("clinvar.2024_01", "01_2024"), - ("clinvar.2015_12", "12_2015"), - ("clinvar.2026_06", "06_2026"), - ("clinvar.2024_00", None), - ("clinvar.2024_13", None), - ("scores", None), - ("clinvar", None), - ("clinvar.2024_01.extra", None), - ("", None), - ], -) -def test_parse_clinvar_namespace(ns, expected): - assert parse_clinvar_namespace(ns) == expected - - @pytest.mark.unit class TestValidateClinvarVariantSummaryDate: def test_valid_past_date(self): diff --git a/tests/lib/csv/__init__.py b/tests/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py new file mode 100644 index 000000000..719794bee --- /dev/null +++ b/tests/lib/csv/test_columns.py @@ -0,0 +1,622 @@ +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.columns import ( + _is_output_null, + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variant_to_csv_row, +) +from tests.helpers.constants import VALID_CALIBRATION_URN + +# --------------------------------------------------------------------------- +# MockVariant +# --------------------------------------------------------------------------- + + +class MockVariant: + """Lightweight mock for Variant used in variant_to_csv_row tests.""" + + def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): + self.urn = urn + self.hgvs_nt = hgvs_nt + self.hgvs_splice = hgvs_splice + self.hgvs_pro = hgvs_pro + self.data = data + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowNullHandling +# --------------------------------------------------------------------------- + + +class TestVariantToCsvRowNullHandling: + """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" + + def test_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"score_data": {}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_score_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"count_data": {}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_count_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_score_data_with_valid_value_preserved(self): + variant = MockVariant(data={"score_data": {"score": 1.5}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "1.5" + + def test_count_data_with_valid_value_preserved(self): + variant = MockVariant(data={"count_data": {"count1": 42}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "42" + + def test_score_data_with_custom_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, na_rep="N/A") + + assert row["score"] == "N/A" + + def test_namespaced_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["scores.score"] == "NA" + + def test_namespaced_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["counts.count1"] == "NA" + + def test_core_columns_with_none_hgvs_uses_na_rep(self): + variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") + columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "NA" + assert row["hgvs_pro"] == "NA" + assert row["hgvs_splice"] == "NA" + assert row["accession"] == "urn:mavedb:00000001-a-1#1" + + def test_mixed_columns_with_missing_data(self): + variant = MockVariant( + hgvs_nt="g.1A>G", + hgvs_pro="p.Met1Val", + data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, + ) + columns = { + "core": ["hgvs_nt", "hgvs_pro"], + "scores": ["score", "se"], + "counts": ["count1", "count2"], + } + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "g.1A>G" + assert row["hgvs_pro"] == "p.Met1Val" + assert row["score"] == "NA" + assert row["se"] == "0.1" + assert row["count1"] == "NA" + assert row["count2"] == "5" + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowUnrecognizedKey +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespace, columns", + [ + ("core", {"core": ["bogus_col"]}), + ("mavedb", {"mavedb": ["bogus_col"]}), + ("vep", {"vep": ["bogus_col"]}), + ("gnomad", {"gnomad": ["bogus_col"]}), + ("clingen", {"clingen": ["bogus_col"]}), + ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), + ("score_set", {"score_set": ["bogus_col"]}), + ("relationship", {"relationship": ["bogus_col"]}), + ("calibration", {f"calibration.{VALID_CALIBRATION_URN}": ["bogus_col"]}), + ], +) +def test_unrecognized_column_key_raises(namespace, columns): + variant = MockVariant() + with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): + variant_to_csv_row(variant, columns) + + +# --------------------------------------------------------------------------- +# TestCalibrationScoreSetAndRelationshipNamespaces +# --------------------------------------------------------------------------- + + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +CALIBRATION_COLUMNS = [ + "title", + "functional_classification", + "acmg_criterion", + "acmg_evidence_strength", + "acmg_evidence_outcome_code", + "pathogenicity_classification", +] + + +class MockTargetGene: + def __init__(self, name): + self.name = name + + +class MockScoreSetForContext: + def __init__(self, urn="urn:mavedb:00000001-a-1", target_gene_names=()): + self.urn = urn + self.target_genes = [MockTargetGene(name) for name in target_gene_names] + + +class MockVariantWithScoreSet(MockVariant): + def __init__(self, score_set=None, **kwargs): + super().__init__(**kwargs) + self.score_set = score_set + + +class TestCalibrationNamespace: + """Tests that a calibration namespace renders a FlatAnnotation, keyed by the calibration's URN.""" + + def test_populated_annotation(self): + annotation = FlatAnnotation( + functional_classification="abnormal", + acmg_criterion="PS3", + acmg_evidence_strength="MODERATE", + acmg_evidence_outcome_code="PS3_moderate", + pathogenicity_classification="PATHOGENIC", + calibration_urn=VALID_CALIBRATION_URN, + calibration_title="Clinical Calibration", + ) + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: CALIBRATION_COLUMNS}, + annotations_by_ns={CALIBRATION_NS: annotation}, + ) + + assert row[f"{CALIBRATION_NS}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS}.functional_classification"] == "abnormal" + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS}.acmg_evidence_strength"] == "MODERATE" + assert row[f"{CALIBRATION_NS}.acmg_evidence_outcome_code"] == "PS3_moderate" + assert row[f"{CALIBRATION_NS}.pathogenicity_classification"] == "PATHOGENIC" + + def test_calibration_columns_are_always_namespaced(self): + """The URN in the header is what disambiguates, so it is kept even for un-namespaced output.""" + annotation = FlatAnnotation(acmg_criterion="PS3") + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={CALIBRATION_NS: annotation}, + namespaced=False, + ) + + assert row == {f"{CALIBRATION_NS}.acmg_criterion": "PS3"} + + def test_no_annotation_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {CALIBRATION_NS: CALIBRATION_COLUMNS}) + + assert all(row[f"{CALIBRATION_NS}.{column}"] == "NA" for column in CALIBRATION_COLUMNS) + + def test_annotation_absent_for_this_namespace_uses_na_rep(self): + other_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={other_ns: FlatAnnotation(acmg_criterion="PS3")}, + ) + + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "NA" + + def test_multiple_calibration_namespaces_are_independent(self): + first_ns = CALIBRATION_NS + second_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {first_ns: ["acmg_criterion"], second_ns: ["acmg_criterion"]}, + annotations_by_ns={ + first_ns: FlatAnnotation(acmg_criterion="PS3"), + second_ns: FlatAnnotation(acmg_criterion="BS3"), + }, + ) + + assert row[f"{first_ns}.acmg_criterion"] == "PS3" + assert row[f"{second_ns}.acmg_criterion"] == "BS3" + + +class TestScoreSetNamespace: + """Tests that the score_set namespace reports the row's score set and target genes.""" + + def test_populated_score_set(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext(target_gene_names=["BRCA1", "BRCA2"])) + columns = {"score_set": ["score_set_urn", "target_gene"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score_set_urn"] == "urn:mavedb:00000001-a-1" + assert row["target_gene"] == "BRCA1; BRCA2" + + def test_empty_collections_use_na_rep(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + row = variant_to_csv_row(variant, {"score_set": ["target_gene"]}) + + assert row["target_gene"] == "NA" + + def test_publication_identifiers_is_not_a_column(self): + """Dropped: it repeats on every row of a score set and score_set_urn already resolves to it.""" + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + with pytest.raises(ValueError, match="unrecognized score_set column: publication_identifiers"): + variant_to_csv_row(variant, {"score_set": ["publication_identifiers"]}) + + +class TestRelationshipNamespace: + """Tests that the relationship namespace reports the caller-supplied match type.""" + + def test_populated_match_type(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}, match_type="exact") + + assert row["match_type"] == "exact" + + def test_missing_match_type_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}) + + assert row["match_type"] == "NA" + + +# --------------------------------------------------------------------------- +# TestRowsToCsv +# --------------------------------------------------------------------------- + + +class TestRowsToCsv: + def test_header_only_when_no_rows(self): + assert rows_to_csv([], ["a", "b"]).splitlines() == ["a,b"] + + def test_writes_rows_in_column_order(self): + rows = [{"b": "2", "a": "1"}, {"a": "3", "b": "4"}] + + assert rows_to_csv(rows, ["a", "b"]).splitlines() == ["a,b", "1,2", "3,4"] + + def test_quotes_values_containing_commas(self): + assert rows_to_csv([{"a": "x,y"}], ["a"]).splitlines()[1] == '"x,y"' + + +# --------------------------------------------------------------------------- +# TestPlanCsvColumns +# --------------------------------------------------------------------------- + + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaces, expected_ns_keys, expected_columns, expected_clinvar", + [ + # `scores` is the one column dataframe validation mandates, nothing more. + (["scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + # The investigator's remaining score columns are their own request token. + (["scores_custom"], {"core", "scores_custom"}, {"scores_custom": ["se", "epsilon"]}, {}), + # Asking for both reproduces the whole score group, `score` first. + ( + ["scores", "scores_custom"], + {"core", "scores", "scores_custom"}, + {"scores": ["score"], "scores_custom": ["se", "epsilon"]}, + {}, + ), + # Counts have no required column, so they are always taken in full. + (["counts"], {"core", "counts"}, {"counts": ["count1", "count2"]}, {}), + ( + ["scores", "counts"], + {"core", "scores", "counts"}, + {"scores": ["score"], "counts": ["count1", "count2"]}, + {}, + ), + (["vep"], {"core", "vep"}, {"vep": ["vep_functional_consequence"]}, {}), + (["gnomad"], {"core", "gnomad"}, {"gnomad": ["gnomad_af"]}, {}), + (["clingen"], {"core", "clingen"}, {"clingen": ["clingen_allele_id"]}, {}), + (["scores", "mavedb"], {"core", "scores", "mavedb"}, {"scores": ["score"]}, {}), + (["clinvar.2024_01"], {"core", "clinvar.2024_01"}, {}, {"clinvar.2024_01": "01_2024"}), + ( + ["clinvar.2024_01", "clinvar.2025_06"], + {"core", "clinvar.2024_01", "clinvar.2025_06"}, + {}, + {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, + ), + # A namespace requested twice is planned once, or it would emit its columns twice. + (["scores", "scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + ], +) +def test_plan_csv_columns(namespaces, expected_ns_keys, expected_columns, expected_clinvar): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces) + + assert set(plan.namespaced_columns.keys()) == expected_ns_keys + assert plan.clinvar_namespaces == expected_clinvar + for namespace, columns in expected_columns.items(): + assert plan.namespaced_columns[namespace] == columns + + assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] + + for ns in expected_clinvar: + assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] + + +def test_plan_csv_columns_reference_hgvs_namespace_populates_columns(): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores", "mavedb"]) + assert plan.namespaced_columns["mavedb"] == [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_digest", + ] + + +# --------------------------------------------------------------------------- +# TestAssembleCsvHeaders +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaced_columns, namespaced, expected", + [ + # Unnamespaced: flat column names + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, + False, + ["accession", "hgvs_nt", "score", "se"], + ), + # Namespaced: scores get prefix, core does not + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, + True, + ["accession", "hgvs_nt", "scores.score"], + ), + # mavedb namespace always gets prefix when namespaced + ( + {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, + True, + ["accession", "mavedb.post_mapped_hgvs_g"], + ), + # ClinVar namespaces always get prefix regardless of namespaced flag + ( + {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, + False, + ["accession", "clinvar.2024_01.clinical_significance"], + ), + # Mixed: respects insertion order + ( + { + "core": ["accession"], + "mavedb": [], + "scores": ["score"], + "clinvar.2024_01": ["clinical_significance"], + }, + True, + ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], + ), + # Empty mavedb namespace when not namespaced produces nothing + ( + {"core": ["hgvs_nt"], "mavedb": []}, + False, + ["hgvs_nt"], + ), + ], +) +def test_assemble_csv_headers(namespaced_columns, namespaced, expected): + assert assemble_csv_headers(namespaced_columns, namespaced) == expected + + +# --------------------------------------------------------------------------- +# TestDropNaColumns +# --------------------------------------------------------------------------- + + +class TestDropNaColumns: + def test_removes_all_na_hgvs_column(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "hgvs_splice" not in new_cols + assert "hgvs_nt" in new_cols + assert "hgvs_pro" in new_cols + for row in new_rows: + assert "hgvs_splice" not in row + + def test_keeps_column_with_some_values(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + def test_does_not_touch_non_hgvs_columns(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "score" in new_cols + assert "hgvs_splice" not in new_cols + + def test_empty_rows_does_not_crash(self): + rows = [] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_rows == [] + assert new_cols == [] + + +def test_plan_csv_columns_omits_reference_hgvs_when_not_requested(): + """It used to be a boolean flag, so the key was always present even when empty.""" + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"]) + + assert "mavedb" not in plan.namespaced_columns + + +# --------------------------------------------------------------------------- +# TestIsOutputNull +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value, expected", + [ + (None, True), + ("", True), + (" ", True), + ("NA", True), + ("na", True), + ("None", True), + ("none", True), + ("NaN", True), + ("nan", True), + ("null", True), + ("NULL", True), + ("nil", True), + ("N/A", True), + ("undefined", True), + ("1.5", False), + ("0", False), + ("hello", False), + ("p.Met1Val", False), + ], +) +def test_is_output_null(value, expected): + assert _is_output_null(value) is expected + + +@pytest.mark.unit +class TestAssembleCsvHeadersRejectsCollisions: + """Un-namespaced output strips the prefix that keeps two namespaces' columns apart. + + The endpoints that ask for un-namespaced output request one namespace each; this holds them to it + rather than letting a future caller discover the problem from a CSV with a column written twice. + """ + + def test_un_namespaced_collision_raises(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "counts": ["score"]}, namespaced=False) + + def test_the_error_names_the_offending_column_and_namespaces(self): + with pytest.raises(ValueError) as excinfo: + assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=False) + + assert "shared" in str(excinfo.value) + assert "scores" in str(excinfo.value) and "counts" in str(excinfo.value) + + def test_namespacing_resolves_what_would_otherwise_collide(self): + headers = assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=True) + + assert headers == ["scores.shared", "counts.shared"] + + def test_scores_and_custom_scores_share_a_prefix_without_colliding(self): + """They emit under one prefix by design, so the guard must not fire on disjoint column sets.""" + headers = assemble_csv_headers({"scores": ["score"], "scores_custom": ["se"]}, namespaced=True) + + assert headers == ["scores.score", "scores.se"] + + def test_a_namespace_sharing_a_prefix_still_collides_on_a_repeated_column(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "scores_custom": ["score"]}, namespaced=True) diff --git a/tests/lib/csv/test_entries.py b/tests/lib/csv/test_entries.py new file mode 100644 index 000000000..7a225c4ef --- /dev/null +++ b/tests/lib/csv/test_entries.py @@ -0,0 +1,64 @@ +import pytest + +from mavedb.lib.csv.entries import clinvar_namespace_entries +from mavedb.lib.csv.namespaces import CsvNamespaceGroup + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar release entries +# +# These go through `clinvar_namespace_entries` rather than asserting how the namespace strings compare, +# because the ordering is what picks the default selection. A test of the comparison alone would keep +# passing if the call site stopped using it. +# --------------------------------------------------------------------------- + + +class TestClinvarNamespaceEntries: + def test_entries_are_ordered_newest_release_first(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.namespace for entry in entries] == [ + "clinvar.2025_10", + "clinvar.2024_11", + "clinvar.2024_02", + ] + + def test_only_the_newest_release_is_selected_by_default(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.selected_by_default for entry in entries] == [True, False, False] + + def test_ordering_survives_uneven_year_widths(self): + """The decay case: as plain strings "clinvar.999_12" sorts above every four-digit year. + + Ordering by the namespace string would put the malformed release first and hand it the default + selection, silently changing which ClinVar call a picker opens with. + """ + entries = clinvar_namespace_entries(["clinvar.999_12", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01", "clinvar.999_12"] + assert entries[0].selected_by_default is True + assert entries[1].selected_by_default is False + + def test_duplicate_releases_are_collapsed(self): + entries = clinvar_namespace_entries(["clinvar.2025_01", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + + def test_unlabelable_namespaces_are_dropped_without_taking_the_default(self): + """An entry that cannot be labelled must not consume the one default slot on its way out.""" + entries = clinvar_namespace_entries(["clinvar.2024_13", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + assert entries[0].selected_by_default is True + + def test_entries_are_grouped_as_annotation(self): + entries = clinvar_namespace_entries(["clinvar.2025_01"]) + + assert entries[0].group is CsvNamespaceGroup.ANNOTATION + assert entries[0].label == "ClinVar significance (January 2025)" + + def test_no_releases_yields_no_entries(self): + assert clinvar_namespace_entries([]) == [] diff --git a/tests/lib/csv/test_namespaces.py b/tests/lib/csv/test_namespaces.py new file mode 100644 index 000000000..a5166ebf4 --- /dev/null +++ b/tests/lib/csv/test_namespaces.py @@ -0,0 +1,242 @@ +import pytest + +from pydantic import TypeAdapter, ValidationError + +from mavedb.lib.csv.namespaces import ( + CSV_NAMESPACE_ERROR_MESSAGE, + STATIC_CSV_NAMESPACES, + CsvNamespace, + CsvNamespaceStr, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_sort_key, + is_valid_csv_namespace, + parse_calibration_namespace, + parse_clinvar_db_version, + parse_clinvar_namespace, +) +from tests.helpers.constants import VALID_CALIBRATION_URN + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", "01_2024"), + ("clinvar.2015_12", "12_2015"), + ("clinvar.2026_06", "06_2026"), + ("clinvar.2024_00", None), + ("clinvar.2024_13", None), + ("scores", None), + ("clinvar", None), + ("clinvar.2024_01.extra", None), + ("", None), + ], +) +def test_parse_clinvar_namespace(ns, expected): + assert parse_clinvar_namespace(ns) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", (2024, 1)), + ("12_2015", (2015, 12)), + ("2024", None), + ("aa_bbbb", None), + ("", None), + ], +) +def test_parse_clinvar_db_version(db_version, expected): + assert parse_clinvar_db_version(db_version) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", "clinvar.2024_01"), + ("11_2024", "clinvar.2024_11"), + ("2_2025", "clinvar.2025_02"), + ("nonsense", None), + ], +) +def test_clinvar_namespace_for_db_version(db_version, expected): + assert clinvar_namespace_for_db_version(db_version) == expected + + +def test_clinvar_namespace_round_trips_through_db_version(): + assert clinvar_namespace_for_db_version(parse_clinvar_namespace("clinvar.2024_01")) == "clinvar.2024_01" + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", (2024, 1)), + ("clinvar.2025_10", (2025, 10)), + # Not a release namespace at all: must sort below every real one rather than raise. + ("scores", (-1, -1)), + ("calibration.urn:mavedb:calibration-abc", (-1, -1)), + ("clinvar.2024_13", (-1, -1)), + ], +) +def test_clinvar_namespace_sort_key(ns, expected): + assert clinvar_namespace_sort_key(ns) == expected + + +def test_clinvar_namespaces_sort_chronologically(): + namespaces = ["clinvar.2024_11", "clinvar.2025_02", "clinvar.2024_02", "clinvar.2025_10"] + + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_10" + assert sorted(namespaces, key=clinvar_namespace_sort_key) == [ + "clinvar.2024_02", + "clinvar.2024_11", + "clinvar.2025_02", + "clinvar.2025_10", + ] + + +def test_sort_key_beats_string_ordering_on_uneven_year_widths(): + """The year group in CLINVAR_NS_PATTERN is unpadded, so string ordering is not chronological. + + This is the decay this key exists to prevent: as plain strings "clinvar.999_12" sorts above + "clinvar.2025_01", which would make a malformed release look like the newest one and hand it the + picker's default selection. + """ + namespaces = ["clinvar.2025_01", "clinvar.999_12"] + + assert max(namespaces) == "clinvar.999_12" + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_01" + + +def test_undatable_namespace_never_sorts_newest(): + assert max(["clinvar.2024_01", "scores"], key=clinvar_namespace_sort_key) == "clinvar.2024_01" + + +# --------------------------------------------------------------------------- +# Calibration namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + (f"calibration.{VALID_CALIBRATION_URN}", VALID_CALIBRATION_URN), + ( + "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + "urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + ), + ("calibration.urn:mavedb:00000001-a-1", None), + ("calibration.not-a-urn", None), + ("calibration.", None), + ("calibration", None), + ("scores", None), + ("", None), + ], +) +def test_parse_calibration_namespace(ns, expected): + assert parse_calibration_namespace(ns) == expected + + +def test_calibration_namespace_round_trips(): + namespace = calibration_namespace_for_urn(VALID_CALIBRATION_URN) + + assert namespace == f"calibration.{VALID_CALIBRATION_URN}" + assert parse_calibration_namespace(namespace) == VALID_CALIBRATION_URN + + +@pytest.mark.parametrize( + "urn", + [ + "urn:mavedb:collection-79471b5b-2dbd-4a96-833c-c33023862437", + "urn:mavedb:00000001-a-1", + "urn:mavedb:calibration-short", + ], +) +def test_non_calibration_urns_do_not_form_valid_namespaces(urn): + assert parse_calibration_namespace(f"calibration.{urn}") is None + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("ns", STATIC_CSV_NAMESPACES) +def test_static_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "clinvar.2024_01", + f"calibration.{VALID_CALIBRATION_URN}", + ], +) +def test_parameterized_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "bogus", + "clinvar", + "clinvar.2024_13", + "calibration", + "calibration.nope", + "SCORES", + "", + ], +) +def test_invalid_namespaces_are_rejected(ns): + assert not is_valid_csv_namespace(ns) + + +def test_static_namespaces_are_the_enum_values(): + """The tuple and the enum must not drift; the tuple feeds the published JSON schema.""" + assert STATIC_CSV_NAMESPACES == tuple(ns.value for ns in CsvNamespace) + + +def test_enum_members_are_usable_as_plain_strings(): + """`plan_csv_columns` keys its column dict by these, so they must behave as their values.""" + assert CsvNamespace.SCORE_SET == "score_set" + assert f"{CsvNamespace.SCORE_SET}" == "score_set" + assert {"score_set": 1}[CsvNamespace.SCORE_SET] == 1 + + +# --------------------------------------------------------------------------- +# CsvNamespaceStr — the validated query-parameter type +# --------------------------------------------------------------------------- + + +_ADAPTER = TypeAdapter(CsvNamespaceStr) + + +@pytest.mark.parametrize( + "ns", + list(STATIC_CSV_NAMESPACES) + ["clinvar.2024_01", f"calibration.{VALID_CALIBRATION_URN}"], +) +def test_validated_type_accepts_valid_namespaces(ns): + assert _ADAPTER.validate_python(ns) == ns + + +@pytest.mark.parametrize("ns", ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.nope", ""]) +def test_validated_type_rejects_invalid_namespaces(ns): + with pytest.raises(ValidationError) as exc_info: + _ADAPTER.validate_python(ns) + + assert CSV_NAMESPACE_ERROR_MESSAGE in str(exc_info.value) + + +def test_error_message_names_the_whole_vocabulary(): + for ns in STATIC_CSV_NAMESPACES: + assert f'"{ns}"' in CSV_NAMESPACE_ERROR_MESSAGE + assert "clinvar.YEAR_MONTH" in CSV_NAMESPACE_ERROR_MESSAGE + assert "calibration." in CSV_NAMESPACE_ERROR_MESSAGE diff --git a/tests/lib/csv/test_specs.py b/tests/lib/csv/test_specs.py new file mode 100644 index 000000000..24ec510ed --- /dev/null +++ b/tests/lib/csv/test_specs.py @@ -0,0 +1,113 @@ +"""The descriptors are what keep planning, row assembly and fetching from drifting apart.""" + +import pytest + +from mavedb.lib.csv.columns import variant_to_csv_row +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from tests.helpers.constants import VALID_CALIBRATION_URN + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + +EVERY_NAMESPACE = [CORE_NAMESPACE, *CsvNamespace, "clinvar.2024_01", CALIBRATION_NS] + + +class _TargetGene: + def __init__(self, name): + self.name = name + + +class _ScoreSet: + urn = "urn:mavedb:00000001-a-1" + target_genes = [_TargetGene("BRCA1")] + + +class _Variant: + """Enough of a Variant for every namespace's resolvers to run against.""" + + urn = "urn:mavedb:00000001-a-1#1" + hgvs_nt = "c.1A>G" + hgvs_splice = None + hgvs_pro = "p.Met1Val" + data = {"score_data": {"score": 1.0}, "count_data": {"count1": 2}} + score_set = _ScoreSet() + + +# --------------------------------------------------------------------------- +# TestNamespaceSpecs +# --------------------------------------------------------------------------- + + +class TestNamespaceSpecs: + def test_every_static_namespace_has_a_spec(self): + """A namespace in the published vocabulary with no descriptor would silently produce no columns.""" + for namespace in CsvNamespace: + assert namespace_spec(namespace) is not None, namespace + + def test_parameterized_namespaces_resolve_to_a_spec(self): + assert namespace_spec("clinvar.2024_01") is not None + assert namespace_spec(CALIBRATION_NS) is not None + assert namespace_spec("not_a_namespace") is None + + @pytest.mark.parametrize("namespace", [ns for ns in CsvNamespace] + ["clinvar.2024_01"]) + def test_every_declared_column_can_be_resolved(self, namespace): + """A column with no resolver raises at row-assembly time, one row into a download.""" + spec = namespace_spec(namespace) + assert spec is not None + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key) is not None, f"{namespace}.{column_key}" + + def test_a_namespace_reading_through_a_relationship_declares_the_fetch_it_needs(self): + """Otherwise the fetch layer would not eager-load it and every row would pay a query.""" + for namespace in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.CLINGEN): + assert namespace_spec(namespace).needs_mappings, namespace + + gnomad = namespace_spec(CsvNamespace.GNOMAD) + assert gnomad.needs_gnomad and gnomad.needs_mappings + + assert namespace_spec(CsvNamespace.SCORE_SET).needs_score_set + calibration = namespace_spec(CALIBRATION_NS) + assert calibration.needs_mappings and calibration.needs_score_set + + def test_resolvers_report_missing_data_rather_than_raising(self): + """Every source is optional on some row: an unmapped variant, a release with no record for it.""" + for namespace in CsvNamespace: + spec = namespace_spec(namespace) + if spec.source in (RowSource.VARIANT, RowSource.MATCH_TYPE): + continue + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key)(None) is None, f"{namespace}.{column_key}" + + +# --------------------------------------------------------------------------- +# TestRowSourceDispatch +# +# A spec names its source; `variant_to_csv_row` is what turns that name into the datum the resolvers are +# called with. That mapping is the one part of the descriptor contract the tests above cannot see, and a +# source with no entry in it raises KeyError at row-assembly time — one row into a download, which is the +# failure mode these descriptors exist to prevent. +# --------------------------------------------------------------------------- + + +class TestRowSourceDispatch: + def test_every_row_source_is_exercised_below(self): + """Guards the parametrization: a new RowSource no namespace here uses would go untested.""" + sources = {namespace_spec(namespace).source for namespace in EVERY_NAMESPACE} + + assert sources == set(RowSource) + + @pytest.mark.parametrize("namespace", EVERY_NAMESPACE) + def test_every_declared_source_resolves_to_a_row_datum(self, namespace): + spec = namespace_spec(namespace) + columns = {namespace: spec.columns(SAMPLE_DATASET_COLUMNS)} + + row = variant_to_csv_row(_Variant(), columns, namespaced=True) + + # Every planned column produced a cell. The parameterized namespaces are passed no per-row datum, + # so theirs are NA — the point here is that the dispatch reached them at all. + assert len(row) == len(columns[namespace]), namespace diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py new file mode 100644 index 000000000..16d52dbc5 --- /dev/null +++ b/tests/lib/csv/test_variant.py @@ -0,0 +1,1160 @@ +# ruff: noqa: E402 + +import csv +import io +from datetime import date +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import event + +from mavedb.lib.csv.namespaces import calibration_namespace_for_urn, is_valid_csv_namespace +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv +from mavedb.lib.csv.variant import ( + BASE_VARIANT_CSV_NAMESPACES, + available_variant_csv_namespaces, + get_variant_csv, +) +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_GNOMAD_DATA_VERSION, + TEST_GNOMAD_VARIANT, + TEST_MINIMAL_MAPPED_VARIANT, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _add_pathogenicity_calibration(db, score_set, variants_in_abnormal_range, urn, title, research_use_only=False): + """Attach a calibration with a normal (BS3) and abnormal (PS3) range to *score_set*. + + Only *variants_in_abnormal_range* are associated with the abnormal range, which is what + ``functional_classification_of_variant`` consults to classify a variant. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=research_use_only, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + abnormal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test abnormal functional range", + description="An abnormal functional range", + functional_classification=FunctionalClassificationOptions.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal_acmg.id, + variants=list(variants_in_abnormal_range), + ) + ) + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test normal functional range", + description="A normal functional range", + functional_classification=FunctionalClassificationOptions.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal_acmg.id, + variants=[], + ) + ) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_rangeless_calibration(db, score_set, urn, title): + """Attach a calibration carrying only a baseline score, with no ranges to classify against. + + It can support neither a functional nor a pathogenicity annotation, so every column of its namespace + would be NA. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=False, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_second_score_set_with_equivalent_variant(db, first_score_set, clingen_allele_id): + """Create a second score set measuring the same ClinGen allele as *first_score_set*'s variant.""" + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, + urn="urn:mavedb:00000001-a-2", + experiment_id=first_score_set.experiment_id, + licence_id=first_score_set.licence_id, + created_by_id=first_score_set.created_by_id, + modified_by_id=first_score_set.modified_by_id, + ) + db.add(score_set) + db.commit() + db.refresh(score_set) + + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + db.add(variant) + db.commit() + db.refresh(variant) + + mapped_variant = MappedVariant( + **TEST_MINIMAL_MAPPED_VARIANT, + variant_id=variant.id, + clingen_allele_id=clingen_allele_id, + ) + db.add(mapped_variant) + db.commit() + db.refresh(mapped_variant) + + return score_set, variant, mapped_variant + + +def _add_clinvar_control(db, mapped_variant, significance, review_status, db_version): + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, + ) + ) + db.add(mapped_variant) + db.commit() + + +def _parse_csv(csv_text): + return list(csv.DictReader(io.StringIO(csv_text))) + + +# --------------------------------------------------------------------------- +# TestGetVariantCsv +# --------------------------------------------------------------------------- + +CALIBRATION_URN_1 = "urn:mavedb:calibration-11111111-1111-1111-1111-111111111111" +CALIBRATION_URN_2 = "urn:mavedb:calibration-22222222-2222-2222-2222-222222222222" +CALIBRATION_URN_OTHER_SCORE_SET = "urn:mavedb:calibration-33333333-3333-3333-3333-333333333333" + +CALIBRATION_NS_1 = calibration_namespace_for_urn(CALIBRATION_URN_1) +CALIBRATION_NS_2 = calibration_namespace_for_urn(CALIBRATION_URN_2) +CALIBRATION_NS_OTHER = calibration_namespace_for_urn(CALIBRATION_URN_OTHER_SCORE_SET) + + +class TestGetVariantCsv: + """Integration tests for the DB-bound clinical CSV composer.""" + + def test_single_variant_yields_one_row_of_base_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["relationship.match_type"] == "exact" + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + assert rows[0]["hgvs_nt"] == TEST_MINIMAL_VARIANT["hgvs_nt"] + + def test_unknown_urn_raises(self, session, setup_lib_db_with_mapped_variant): + with pytest.raises(ValueError, match="not found"): + get_variant_csv(session, "urn:mavedb:00000001-a-1#999") + + def test_no_calibration_yields_no_calibration_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + csv_text = get_variant_csv(session, variant.urn) + + assert "calibration." not in csv_text + + def test_calibration_namespace_is_included_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Test Clinical Calibration" + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "abnormal" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "STRONG" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + + def test_variant_outside_calibration_ranges_is_uncertain(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "indeterminate" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_multiple_calibrations_appear_side_by_side(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Contains The Variant" + ) + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Does Not Contain The Variant" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_2}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_research_use_only_calibration_is_offered_but_labelled(self, session, setup_lib_db_with_mapped_variant): + """The score set page already shows these, so the export offers them — flagged, not hidden.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_variant_csv_namespaces(session, variant.urn) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + + def test_research_use_only_calibration_is_not_in_the_default_download( + self, session, setup_lib_db_with_mapped_variant + ): + """A clinically-framed default must not silently carry unvalidated thresholds.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + assert CALIBRATION_NS_1 not in get_variant_csv(session, variant.urn) + + def test_research_use_only_calibration_is_served_when_named(self, session, setup_lib_db_with_mapped_variant): + """Naming the namespace is the opt-in, and the exported row declares its own standing.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "True" + + def test_clinical_calibration_declares_it_is_not_research_use_only(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_explicit_namespaces_restrict_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Requested") + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_2, title="Omitted") + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1]) + rows = _parse_csv(csv_text) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Requested" + assert CALIBRATION_NS_2 not in csv_text + # Namespaces the caller did not ask for contribute no columns. + assert "gnomad.gnomad_af" not in csv_text + assert "relationship.match_type" not in csv_text + + def test_equivalent_measurements_share_clingen_allele_id(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # The requested measurement comes first. + assert rows[0]["accession"] == variant.urn + assert rows[1]["score_set.score_set_urn"] == "urn:mavedb:00000001-a-2" + assert all(row["relationship.match_type"] == "exact" for row in rows) + assert all(row["clingen.clingen_allele_id"] == "CA123456" for row in rows) + + def test_each_measurement_is_interpreted_only_under_its_own_calibrations( + self, session, setup_lib_db_with_mapped_variant + ): + """A score from one assay carries no meaning under another assay's thresholds.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Score Set Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Score Set Calibration", + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # Each row is classified under its own score set's calibration... + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[1][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "PATHOGENIC" + # ...and left empty under the other score set's. + assert rows[0][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "NA" + assert rows[1][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_calibration_entries_report_their_score_set(self, session, setup_lib_db_with_mapped_variant): + """A calibration means nothing against another score set's scores, so say which one owns it.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Assay Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Assay Calibration", + ) + + by_namespace = {entry.namespace: entry for entry in available_variant_csv_namespaces(session, variant.urn)} + + first, second = by_namespace[CALIBRATION_NS_1], by_namespace[CALIBRATION_NS_OTHER] + assert first.score_set.urn == variant.score_set.urn + assert second.score_set.urn == other_score_set.urn + # The title is carried too, so a picker can name the score set rather than show its URN. + assert first.score_set.title == variant.score_set.title + # The two are distinguishable, which is the whole point. + assert first.score_set.urn != second.score_set.urn + + def test_non_calibration_entries_have_no_owning_score_set(self, session, setup_lib_db_with_mapped_variant): + """gnomAD and friends apply to any measurement, so there is no score set to attribute them to.""" + variant = setup_lib_db_with_mapped_variant.variant + + entries = available_variant_csv_namespaces(session, variant.urn) + + assert all(entry.score_set is None for entry in entries if not entry.namespace.startswith("calibration.")) + + def test_variant_without_clingen_allele_id_stands_alone(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + # A second measurement with a null allele ID must not be pulled in on a null match. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, None) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["clingen.clingen_allele_id"] == "NA" + + def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" + mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" + mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" + mapped_variant.vep_functional_consequence = "missense_variant" + mapped_variant.clingen_allele_id = "CA123456" + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + # Patched where it is used, not where it is defined: `fetch` binds the value with a `from` + # import, so patching `mavedb.lib.gnomad` would leave the query filtering on the real version. + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" + assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" + assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NP_000305.3:p.Lys34Glu" + assert rows[0]["vep.vep_functional_consequence"] == "missense_variant" + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["clingen.clingen_allele_id"] == "CA123456" + + def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", "v9.9"): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + # The variant still gets a row: the version predicate is in the join's ON clause, so a gnomAD + # record from another version leaves the frequency NA rather than dropping the variant. + assert len(rows) == 1 + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "reviewed by expert panel", "02_2025") + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + rows = _parse_csv(csv_text) + + # The release is carried in the column name so the call stays citable. + assert "clinvar.2025_02.clinical_significance" in csv_text.splitlines()[0] + assert rows[0]["clinvar.2025_02.clinical_significance"] == "Pathogenic" + assert rows[0]["clinvar.2025_02.clinical_review_status"] == "reviewed by expert panel" + assert "clinvar.2024_11" not in csv_text + + def test_non_clinvar_control_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="ABC123", + gene_symbol="BRCA1", + clinical_significance="benign", + clinical_review_status="lots of convincing evidence", + db_name="GenDB", + db_version="2024", + ) + ) + session.add(mapped_variant) + session.commit() + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + + assert "clinvar" not in csv_text + assert "benign" not in csv_text + + def test_provenance_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + session.add(TargetGene(score_set_id=variant.score_set.id, name="PTEN", category="protein_coding")) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["score_set.target_gene"] == "PTEN" + + def test_unmapped_variant_in_an_unmapped_score_set_omits_mapping_columns(self, session, setup_lib_db_with_variant): + """Nothing in this score set has been mapped, so those columns would say nothing at all.""" + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + for omitted in ( + "mavedb.post_mapped_hgvs_g", + "clingen.clingen_allele_id", + "gnomad.gnomad_af", + "vep.vep_functional_consequence", + ): + assert omitted not in csv_text + + def test_unmapped_variant_in_a_mapped_score_set_keeps_mapping_columns_as_na( + self, session, setup_lib_db_with_variant + ): + """The score set is mapped, so the columns exist for it; NA is the honest value for this variant.""" + variant = setup_lib_db_with_variant + mapped_sibling = Variant( + **{**TEST_MINIMAL_VARIANT, "urn": f"{variant.score_set.urn}#2"}, score_set_id=variant.score_set_id + ) + session.add(mapped_sibling) + session.commit() + session.add(MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_sibling.id)) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NA" + assert rows[0]["clingen.clingen_allele_id"] == "NA" + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_unmapped_variant_respects_requested_namespaces(self, session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores"]) + + assert "gnomad.gnomad_af" not in csv_text + assert "scores.score" in csv_text.splitlines()[0] + + def test_superseded_mapping_is_ignored(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.current = False + session.add(mapped_variant) + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, + variant_id=mapped_variant.variant_id, + clingen_allele_id="CA999999", + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert len(rows) == 1 + assert rows[0]["clingen.clingen_allele_id"] == "CA999999" + + def test_does_not_load_whole_score_range_variant_collections(self, session, setup_lib_db_with_mapped_variant): + """Range membership must come from the association table, not by loading every variant of a range. + + The ORM check in ``annotation.classification`` loads each range's entire variant collection — with + every variant's score data — once per range per row. On a large score set that dominates the + export's runtime, so this pins the cheap path rather than trusting it to stay. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Second") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + get_variant_csv(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + collection_loads = [ + statement + for statement in statements + if "score_calibration_functional_classification_variants" in statement and "variants.data" in statement + ] + assert collection_loads == [], ( + f"{len(collection_loads)} range-collection load(s) during one export; " + "membership should come from the association table" + ) + + def test_namespace_discovery_does_not_scan_score_set_variants(self, session, setup_lib_db_with_mapped_variant): + """Discovery must key on score sets, not join through their variants. + + `ScoreSet.variants` multiplies the calibration join by every variant in the score set before + DISTINCT collapses it again, which made this route far slower than the score-set equivalent that + filters on score_set_id directly. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + available_variant_csv_namespaces(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + calibration_scans = [ + statement + for statement in statements + if "score_calibrations" in statement and " variants" in statement.replace("\n", " ") + ] + assert ( + calibration_scans == [] + ), "calibration discovery joined the variants table; it should filter on score_set_id" + + def test_base_namespaces_are_all_present_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + header = _parse_csv(get_variant_csv(session, variant.urn))[0].keys() + + # One representative column per base namespace. + for column in ( + "scores.score", + "vep.vep_functional_consequence", + "gnomad.gnomad_af", + "clingen.clingen_allele_id", + "score_set.score_set_urn", + "relationship.match_type", + ): + assert column in header, f"{column} missing; BASE_VARIANT_CSV_NAMESPACES={BASE_VARIANT_CSV_NAMESPACES}" + + +# --------------------------------------------------------------------------- +# TestComputeAvailableCsvNamespaces +# --------------------------------------------------------------------------- + + +class TestComputeAvailableCsvNamespaces: + """What a namespace selector is offered for a score set.""" + + def test_mapped_score_set_offers_mapping_backed_namespaces(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert {"vep", "gnomad", "clingen"} <= set(namespaces) + + def test_unmapped_score_set_omits_mapping_backed_namespaces(self, session, setup_lib_db_with_variant): + score_set = setup_lib_db_with_variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert not {"vep", "gnomad", "clingen"} & set(namespaces) + + def test_relationship_is_never_offered(self, session, setup_lib_db_with_mapped_variant): + """match_type describes a row's relation to a requested record, which a score set has no notion of.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + assert "relationship" not in [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set) + ] + + def test_score_and_count_namespaces_follow_dataset_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["scores.score"], "count_columns": []} + session.add(score_set) + session.commit() + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "scores" in namespaces + assert "counts" not in namespaces + + def test_calibration_namespaces_are_offered(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, variant.score_set)] + + assert CALIBRATION_NS_1 in namespaces + + def test_research_use_only_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + # Reported in its own right, not left to be inferred from the label or from the unchecked box: + # this is the one reason for unchecking that decides whether the data may be published. + assert entry.research_use_only is True + + def test_clinical_calibration_is_offered_checked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Clinical Calibration" + assert entry.selected_by_default is True + assert entry.research_use_only is False + assert entry.score_set.urn == variant.score_set.urn + + def test_rangeless_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + """The score-set export covers the score set's own calibrations, so a rangeless one is still + requestable — but it would contribute nothing except NA, so it must not open checked and must not + reach the public dump, which takes only what discovery selects by default. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Baseline Only" + assert entry.selected_by_default is False + + def test_variant_discovery_omits_a_rangeless_calibration_entirely(self, session, setup_lib_db_with_mapped_variant): + """A variant's calibrations are scoped to what interprets this allele; one that interprets + nothing is not a choice worth offering. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_clinvar_namespaces_are_offered_per_release(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", "02_2025") + + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + ] + + assert "clinvar.2024_11" in namespaces + assert "clinvar.2025_02" in namespaces + + def test_only_the_newest_clinvar_release_is_selected_by_default(self, session, setup_lib_db_with_mapped_variant): + """MaveDB carries around ten releases; a picker opening with all of them checked is unusable.""" + mapped_variant = setup_lib_db_with_mapped_variant + for db_version in ("11_2024", "02_2025", "06_2024"): + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", db_version) + + by_namespace = { + entry.namespace: entry + for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + } + + assert by_namespace["clinvar.2025_02"].selected_by_default is True + assert by_namespace["clinvar.2024_11"].selected_by_default is False + assert by_namespace["clinvar.2024_06"].selected_by_default is False + # The older releases are still on offer — comparing a call across releases is a real thing to want. + assert len([ns for ns in by_namespace if ns.startswith("clinvar.")]) == 3 + + def test_every_offered_namespace_is_valid(self, session, setup_lib_db_with_mapped_variant): + """Discovery must only advertise namespaces the endpoints will actually accept.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "02_2025") + + entries = available_score_set_csv_namespaces(session, variant.score_set) + + assert entries + assert all(is_valid_csv_namespace(entry.namespace) for entry in entries) + # Every entry must also be presentable, or a picker has nothing to render. + assert all(entry.label for entry in entries) + assert all(entry.group for entry in entries) + + def test_entries_are_labeled_for_a_picker(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Brnich et al. 2019" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "11_2024") + + by_namespace = { + entry.namespace: entry for entry in available_score_set_csv_namespaces(session, variant.score_set) + } + + # A calibration is named by its title, not its URN. + assert by_namespace[CALIBRATION_NS_1].label == "Brnich et al. 2019" + assert by_namespace[CALIBRATION_NS_1].group == "calibration" + # A ClinVar release is named by its date. + assert by_namespace["clinvar.2024_11"].label == "ClinVar significance (November 2024)" + assert by_namespace["clinvar.2024_11"].group == "annotation" + assert by_namespace["gnomad"].label == "gnomAD allele frequency" + assert by_namespace["score_set"].group == "provenance" + + +# --------------------------------------------------------------------------- +# TestAnchorMappingIsDeterministic +# --------------------------------------------------------------------------- + + +class TestAnchorMappingIsDeterministic: + """Everything in the CSV follows from which current mapping anchors the request.""" + + def test_repeat_downloads_agree_when_several_mappings_claim_to_be_current( + self, session, setup_lib_db_with_mapped_variant + ): + """Nothing in the schema stops two rows from being current, so the pick must not be arbitrary.""" + variant = setup_lib_db_with_mapped_variant.variant + + newer = MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2030, 1, 1), "clingen_allele_id": "CA_NEWER"}, + variant_id=variant.id, + ) + session.add(newer) + session.commit() + + first = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + second = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + + assert first == second + # The requested variant anchors the export and comes first, so this pins which mapping was + # picked rather than merely whether the newer one appears anywhere in the output. + assert first[0]["clingen.clingen_allele_id"] == "CA_NEWER" + + def test_a_variant_with_two_current_mappings_is_reported_once(self, session, setup_lib_db_with_mapped_variant): + """Two current mappings on one variant are the same measurement twice, not two equivalents. + + Emitting both would also break the row ordering downstream, which restores the caller's order from + the variant ids alone. + """ + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=variant.id, + ) + ) + session.commit() + + # A genuine equivalent in another score set, so the widening is doing something to dedupe within. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, "urn:mavedb:00000001-a-2#1"] + + def test_an_equivalent_variants_extra_current_mapping_is_reported_once( + self, session, setup_lib_db_with_mapped_variant + ): + """The same rule applies to the widened rows, not just to the anchor.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _, other_variant, _ = _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=other_variant.id, + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, other_variant.urn] + + +# --------------------------------------------------------------------------- +# TestScoreSetCsvCalibrationColumns +# --------------------------------------------------------------------------- + + +class TestScoreSetCsvCalibrationColumns: + """The score-set CSV must fill the calibration columns its discovery advertises.""" + + def test_calibration_columns_are_populated(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_variant_outside_the_range_is_uncertain_not_blank(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Clinical") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + + def test_rangeless_calibration_reports_its_identity_with_no_interpretation( + self, session, setup_lib_db_with_mapped_variant + ): + """It cannot classify anything, but which calibration was consulted is still on the record. + + The public dump carries these namespaces, so a wholly-NA block would be the archive claiming to + know less than the database does. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Baseline Only" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + assert row[f"{CALIBRATION_NS_1}.functional_classification"] == "NA" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_everything_discovery_advertises_is_actually_populated(self, session, setup_lib_db_with_mapped_variant): + """Discovery and the export must agree, or a dump ships documented but empty columns.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + advertised = [ + entry.namespace + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace.startswith("calibration.") + ] + assert advertised, "no calibration namespace advertised; the rest of this test proves nothing" + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores"] + advertised, namespaced=True) + ) + row = next(r for r in rows if r["accession"] == variant.urn) + + for namespace in advertised: + assert row[f"{namespace}.title"] != "NA", f"{namespace} advertised but its columns are empty" + + def test_counts_are_always_taken_in_full(self, session, setup_lib_db_with_mapped_variant): + """Counts have no required column, so nothing narrows them the way ``scores`` is narrowed.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score"], "count_columns": ["c_0"]} + session.add(score_set) + session.commit() + + csv_text = get_score_set_variants_as_csv(session, score_set, ["counts"], namespaced=True) + + assert "counts.c_0" in csv_text.splitlines()[0] + + +class TestPrivateCalibrationsAreNotDisclosed: + """A calibration's READ permission is stricter than its score set's. + + Private ones are readable only by their owner, by contributors when investigator-provided, or by an + admin, so reading the measurement does not entitle a caller to the interpretation. + """ + + @pytest.fixture + def private_calibration(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + calibration = _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Unpublished Calibration" + ) + calibration.private = True + session.add(calibration) + session.commit() + return calibration + + def test_score_set_discovery_omits_it_by_default(self, session, private_calibration): + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, private_calibration.score_set) + ] + + assert CALIBRATION_NS_1 not in namespaces + + def test_variant_discovery_omits_it_by_default( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + variant = setup_lib_db_with_mapped_variant.variant + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_naming_the_urn_directly_yields_no_interpretation( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + """Discovery is not the gate: a caller who knows the URN must still be refused the data.""" + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn, ["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_score_set_csv_withholds_it_too(self, session, private_calibration): + rows = _parse_csv( + get_score_set_variants_as_csv( + session, private_calibration.score_set, ["scores", CALIBRATION_NS_1], namespaced=True + ) + ) + + assert all(row[f"{CALIBRATION_NS_1}.title"] == "NA" for row in rows) + + def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_mapped_variant, private_calibration): + """The predicate widens access; it must not be a blanket ban on private calibrations.""" + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv( + get_variant_csv( + session, + variant.urn, + ["scores", CALIBRATION_NS_1], + may_read_calibration=lambda calibration: True, + ) + ) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Unpublished Calibration" + + def test_the_public_export_never_carries_it(self, session, private_calibration): + """The dump has no caller, so the default must be the public subset.""" + from mavedb.scripts.export_public_data import annotation_export_namespaces + + assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set) + + +class TestScoreColumnNamespaces: + """`scores` is the required column; `scores_custom` is the rest, emitted under the same prefix.""" + + @pytest.fixture + def score_set_with_custom_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score", "se"], "count_columns": []} + session.add(score_set) + session.commit() + return score_set + + def test_scores_alone_emits_only_the_required_column(self, session, score_set_with_custom_columns): + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores"], namespaced=True) + )[0] + + assert "scores.score" in header + assert "scores.se" not in header + + def test_custom_columns_are_emitted_under_the_scores_prefix(self, session, score_set_with_custom_columns): + """The published header must not change: `scores_custom` is a request token, not a column prefix.""" + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores_custom"], namespaced=True) + )[0] + + assert "scores.se" in header + assert not any(column.startswith("scores_custom.") for column in header) + + def test_both_namespaces_reproduce_the_whole_score_group_in_order(self, session, score_set_with_custom_columns): + header = list( + _parse_csv( + get_score_set_variants_as_csv( + session, score_set_with_custom_columns, ["scores", "scores_custom"], namespaced=True + ) + )[0] + ) + + assert [column for column in header if column.startswith("scores.")] == ["scores.score", "scores.se"] + + def test_discovery_offers_custom_columns_only_when_there_are_any( + self, session, score_set_with_custom_columns, setup_lib_db_with_mapped_variant + ): + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" in offered + + score_set_with_custom_columns.dataset_columns = {"score_columns": ["score"], "count_columns": []} + session.add(score_set_with_custom_columns) + session.commit() + + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" not in offered diff --git a/tests/lib/mave/test_utils.py b/tests/lib/mave/test_utils.py deleted file mode 100644 index 44716595b..000000000 --- a/tests/lib/mave/test_utils.py +++ /dev/null @@ -1,31 +0,0 @@ -import pytest - -from mavedb.lib.mave.utils import is_csv_output_null - - -@pytest.mark.unit -@pytest.mark.parametrize( - "value, expected", - [ - (None, True), - ("", True), - (" ", True), - ("NA", True), - ("na", True), - ("None", True), - ("none", True), - ("NaN", True), - ("nan", True), - ("null", True), - ("NULL", True), - ("nil", True), - ("N/A", True), - ("undefined", True), - ("1.5", False), - ("0", False), - ("hello", False), - ("p.Met1Val", False), - ], -) -def test_is_csv_output_null(value, expected): - assert bool(is_csv_output_null(value)) is expected diff --git a/tests/lib/test_acmg.py b/tests/lib/test_acmg.py index cc5dfac0c..bf82e3629 100644 --- a/tests/lib/test_acmg.py +++ b/tests/lib/test_acmg.py @@ -6,6 +6,7 @@ pytest.importorskip("psycopg2") from mavedb.lib.acmg import ( + acmg_evidence_outcome_code, ACMGCriterion, StrengthOfEvidenceProvided, find_or_create_acmg_classification, @@ -241,3 +242,36 @@ def test_find_or_create_acmg_classification_does_not_commit(session): ).scalar_one_or_none() assert existing is None + + +######################################################################################################################## +# Tests for acmg_evidence_outcome_code +######################################################################################################################## + + +@pytest.mark.parametrize( + "criterion, evidence_strength, expected", + [ + # STRONG is a criterion's baseline, so it is written bare. + ("PS3", "STRONG", "PS3"), + ("BS3", "STRONG", "BS3"), + # Anything else is suffixed. + ("PS3", "VERY_STRONG", "PS3_very_strong"), + ("PS3", "MODERATE", "PS3_moderate"), + ("PS3", "SUPPORTING", "PS3_supporting"), + ("BS3", "SUPPORTING", "BS3_supporting"), + # MaveDB's intermediate strength has no VA-Spec equivalent, but the code format is the same. + ("PS3", "MODERATE_PLUS", "PS3_moderate_plus"), + # No strength means the criterion was evaluated and not met. + ("PS3", None, "PS3_not_met"), + ("BS3", None, "BS3_not_met"), + ], +) +def test_acmg_evidence_outcome_code(criterion, evidence_strength, expected): + assert acmg_evidence_outcome_code(criterion, evidence_strength) == expected + + +def test_acmg_evidence_outcome_code_is_case_insensitive_about_strength(): + """Callers pass a name from whichever enumeration they hold; casing should not change the result.""" + assert acmg_evidence_outcome_code("PS3", "strong") == "PS3" + assert acmg_evidence_outcome_code("PS3", "Moderate") == "PS3_moderate" diff --git a/tests/lib/test_score_set_csv.py b/tests/lib/test_score_set_csv.py deleted file mode 100644 index 9ec913090..000000000 --- a/tests/lib/test_score_set_csv.py +++ /dev/null @@ -1,433 +0,0 @@ -import pytest - -from mavedb.lib.score_set_csv import ( - assemble_csv_headers, - drop_na_columns_from_csv_file_rows, - plan_csv_columns, - variant_to_csv_row, -) - -# --------------------------------------------------------------------------- -# MockVariant -# --------------------------------------------------------------------------- - - -class MockVariant: - """Lightweight mock for Variant used in variant_to_csv_row tests.""" - - def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): - self.urn = urn - self.hgvs_nt = hgvs_nt - self.hgvs_splice = hgvs_splice - self.hgvs_pro = hgvs_pro - self.data = data - - -# --------------------------------------------------------------------------- -# TestVariantToCsvRowNullHandling -# --------------------------------------------------------------------------- - - -class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" - - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "1.5" - - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "42" - - def test_score_data_with_custom_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, na_rep="N/A") - - assert row["score"] == "N/A" - - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): - variant = MockVariant( - hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", - data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, - ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" - - -# --------------------------------------------------------------------------- -# TestVariantToCsvRowUnrecognizedKey -# --------------------------------------------------------------------------- - - -@pytest.mark.unit -@pytest.mark.parametrize( - "namespace, columns", - [ - ("core", {"core": ["bogus_col"]}), - ("mavedb", {"mavedb": ["bogus_col"]}), - ("vep", {"vep": ["bogus_col"]}), - ("gnomad", {"gnomad": ["bogus_col"]}), - ("clingen", {"clingen": ["bogus_col"]}), - ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), - ], -) -def test_unrecognized_column_key_raises(namespace, columns): - variant = MockVariant() - with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): - variant_to_csv_row(variant, columns) - - -# --------------------------------------------------------------------------- -# TestPlanCsvColumns -# --------------------------------------------------------------------------- - - -SAMPLE_DATASET_COLUMNS = { - "score_columns": ["score", "se", "epsilon"], - "count_columns": ["count1", "count2"], -} - - -@pytest.mark.unit -@pytest.mark.parametrize( - "namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar", - [ - # scores-only - ( - ["scores"], - {}, - {"core", "mavedb", "scores"}, - ["score", "se", "epsilon"], - {}, - ), - # counts-only - ( - ["counts"], - {}, - {"core", "mavedb", "counts"}, - None, - {}, - ), - # both scores and counts - ( - ["scores", "counts"], - {}, - {"core", "mavedb", "scores", "counts"}, - ["score", "se", "epsilon"], - {}, - ), - # vep adds its column - ( - ["vep"], - {}, - {"core", "mavedb", "vep"}, - None, - {}, - ), - # gnomad adds its column - ( - ["gnomad"], - {}, - {"core", "mavedb", "gnomad"}, - None, - {}, - ), - # clingen adds its column - ( - ["clingen"], - {}, - {"core", "mavedb", "clingen"}, - None, - {}, - ), - # include_custom_columns=False -> only REQUIRED_SCORE_COLUMN for scores - ( - ["scores"], - {"include_custom_columns": False}, - {"core", "mavedb", "scores"}, - ["score"], - {}, - ), - # include_post_mapped_hgvs populates mavedb namespace - ( - ["scores"], - {"include_post_mapped_hgvs": True}, - {"core", "mavedb", "scores"}, - ["score", "se", "epsilon"], - {}, - ), - # single ClinVar namespace - ( - ["clinvar.2024_01"], - {}, - {"core", "mavedb", "clinvar.2024_01"}, - None, - {"clinvar.2024_01": "01_2024"}, - ), - # multiple ClinVar versions - ( - ["clinvar.2024_01", "clinvar.2025_06"], - {}, - {"core", "mavedb", "clinvar.2024_01", "clinvar.2025_06"}, - None, - {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, - ), - ], -) -def test_plan_csv_columns(namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar): - plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces, **kwargs) - - assert set(plan.namespaced_columns.keys()) == expected_ns_keys - assert plan.clinvar_namespaces == expected_clinvar - - if expected_score_cols is not None: - assert plan.namespaced_columns["scores"] == expected_score_cols - - # core always has the standard 4 columns - assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] - - # vep, gnomad, clingen get their fixed columns when present - if "vep" in plan.namespaced_columns: - assert plan.namespaced_columns["vep"] == ["vep_functional_consequence"] - if "gnomad" in plan.namespaced_columns: - assert plan.namespaced_columns["gnomad"] == ["gnomad_af"] - if "clingen" in plan.namespaced_columns: - assert plan.namespaced_columns["clingen"] == ["clingen_allele_id"] - - # ClinVar namespaces get their standard columns - for ns in expected_clinvar: - assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] - - -def test_plan_csv_columns_post_mapped_hgvs_populates_mavedb(): - plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"], include_post_mapped_hgvs=True) - assert plan.namespaced_columns["mavedb"] == [ - "post_mapped_hgvs_g", - "post_mapped_hgvs_p", - "post_mapped_hgvs_c", - "post_mapped_hgvs_at_assay_level", - "post_mapped_vrs_digest", - ] - - -# --------------------------------------------------------------------------- -# TestAssembleCsvHeaders -# --------------------------------------------------------------------------- - - -@pytest.mark.unit -@pytest.mark.parametrize( - "namespaced_columns, namespaced, expected", - [ - # Unnamespaced: flat column names - ( - {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, - False, - ["accession", "hgvs_nt", "score", "se"], - ), - # Namespaced: scores get prefix, core does not - ( - {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, - True, - ["accession", "hgvs_nt", "scores.score"], - ), - # mavedb namespace always gets prefix when namespaced - ( - {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, - True, - ["accession", "mavedb.post_mapped_hgvs_g"], - ), - # ClinVar namespaces always get prefix regardless of namespaced flag - ( - {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, - False, - ["accession", "clinvar.2024_01.clinical_significance"], - ), - # Mixed: respects insertion order - ( - { - "core": ["accession"], - "mavedb": [], - "scores": ["score"], - "clinvar.2024_01": ["clinical_significance"], - }, - True, - ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], - ), - # Empty mavedb namespace when not namespaced produces nothing - ( - {"core": ["hgvs_nt"], "mavedb": []}, - False, - ["hgvs_nt"], - ), - ], -) -def test_assemble_csv_headers(namespaced_columns, namespaced, expected): - assert assemble_csv_headers(namespaced_columns, namespaced) == expected - - -# --------------------------------------------------------------------------- -# TestDropNaColumns -# --------------------------------------------------------------------------- - - -class TestDropNaColumns: - def test_removes_all_na_hgvs_column(self): - rows = [ - {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, - {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, - ] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert "hgvs_splice" not in new_cols - assert "hgvs_nt" in new_cols - assert "hgvs_pro" in new_cols - for row in new_rows: - assert "hgvs_splice" not in row - - def test_keeps_column_with_some_values(self): - rows = [ - {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, - {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, - ] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - def test_does_not_touch_non_hgvs_columns(self): - rows = [ - {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, - ] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert "score" in new_cols - assert "hgvs_splice" not in new_cols - - def test_empty_rows_does_not_crash(self): - rows = [] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert new_rows == [] - assert new_cols == [] diff --git a/tests/lib/test_urns.py b/tests/lib/test_urns.py new file mode 100644 index 000000000..380bf0fc9 --- /dev/null +++ b/tests/lib/test_urns.py @@ -0,0 +1,99 @@ +import pytest + +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# Each of these pairs is one a lexical sort gets wrong. They are the reason the keys exist, so they are +# asserted against plain string ordering too — a test that only checked the key would keep passing if +# someone decided the URNs sort fine on their own. +# --------------------------------------------------------------------------- + + +class TestScoreSetUrnSortKey: + def test_unpadded_score_set_number_orders_numerically(self): + urns = ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + + assert sorted(urns) == ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-a-2", + "urn:mavedb:00000001-a-10", + ] + + def test_experiment_suffix_orders_by_length_then_alphabetically(self): + """MaveDB assigns experiment suffixes a..z then aa..az, so `z` precedes `aa`.""" + urns = ["urn:mavedb:00000001-aa-1", "urn:mavedb:00000001-b-1", "urn:mavedb:00000001-z-1"] + + assert sorted(urns)[0] == "urn:mavedb:00000001-aa-1" + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-b-1", + "urn:mavedb:00000001-z-1", + "urn:mavedb:00000001-aa-1", + ] + + def test_experiment_sets_order_before_their_experiments(self): + urns = ["urn:mavedb:00000002-a-1", "urn:mavedb:00000001-z-9"] + + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-z-9", + "urn:mavedb:00000002-a-1", + ] + + def test_zero_experiment_suffix_is_accepted(self): + """The experiment URN grammar allows a literal `0` alongside the letter suffixes.""" + assert score_set_urn_sort_key("urn:mavedb:00000001-0-1")[0] == 0 + + @pytest.mark.parametrize( + "urn", + [ + # An unpublished score set. A SQL cast on the suffix would be handed "467a" and error. + "tmp:8f14e45f-ceea-467a-9c4f-0b1d2e3f4a5b", + "not a urn at all", + "", + None, + ], + ) + def test_undecomposable_urns_sort_after_published_ones_without_raising(self, urn): + published = "urn:mavedb:00000001-a-1" + + assert sorted([urn, published], key=score_set_urn_sort_key) == [published, urn] + + def test_undecomposable_urns_still_order_deterministically_among_themselves(self): + urns = ["tmp:b", "tmp:a", "tmp:c"] + + assert sorted(urns, key=score_set_urn_sort_key) == ["tmp:a", "tmp:b", "tmp:c"] + + +class TestVariantUrnSortKey: + def test_unpadded_variant_number_orders_numerically(self): + urns = [f"urn:mavedb:00000001-a-1#{n}" for n in (2, 10, 1)] + + assert sorted(urns)[0] == "urn:mavedb:00000001-a-1#1" + assert sorted(urns)[1] == "urn:mavedb:00000001-a-1#10" + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#1", + "urn:mavedb:00000001-a-1#2", + "urn:mavedb:00000001-a-1#10", + ] + + def test_variants_group_by_score_set_before_their_number(self): + urns = ["urn:mavedb:00000001-a-2#1", "urn:mavedb:00000001-a-1#3"] + + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#3", + "urn:mavedb:00000001-a-2#1", + ] + + def test_unpublished_variant_urn_still_orders_by_its_number(self): + """A variant of an unpublished score set is `tmp:#N`, so the suffix is still parseable.""" + urns = ["tmp:abc#10", "tmp:abc#2"] + + assert sorted(urns, key=variant_urn_sort_key) == ["tmp:abc#2", "tmp:abc#10"] + + @pytest.mark.parametrize("urn", ["urn:mavedb:00000001-a-1", "", None]) + def test_urns_without_a_variant_suffix_sort_last_without_raising(self, urn): + numbered = "urn:mavedb:00000001-a-1#1" + + assert sorted([urn, numbered], key=variant_urn_sort_key) == [numbered, urn] From 39cfd21e4b4401c0ea9b33db8b0492d6088ef011 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 10:37:31 -0700 Subject: [PATCH 3/9] feat(api): add variant-level clinical CSV and namespace discovery endpoints Add GET /variants/{urn}/csv and GET /variants/{urn}/csv-namespaces, mirroring the score-set CSV endpoints but scoped to a single variant. The variant endpoint widens over the variant's equivalent measurements (as the annotation endpoints already do), so a calibration belonging to another score set that also measured this allele is included too, and emits one row per current measurement with the requested variant first. Add GET /score-sets/{urn}/csv-namespaces alongside it, and switch GET /score-sets/{urn}/variants/data to build its namespace list and validation from discovery instead of a hand-maintained _VALID_STATIC_NAMESPACES set and the ClinVar regex check. - Accept drop_unused_hgvs_columns on the score-set CSV endpoints, keeping drop_na_columns, include_post_mapped_hgvs, and include_custom_columns working as deprecated query params via resolve_deprecated_csv_params - Ask calibration READ permission separately from score-set READ on every CSV endpoint, since a private calibration's interpretation is not implied by being able to read the measurement it applies to - /score-sets/{urn}/scores now includes both score namespaces ("scores" plus "scores_custom"), matching its historical behavior of returning every score column the investigator uploaded - Make _stream_generated_annotations resilient to a single variant's annotation failing mid-stream: log and emit it as unannotated rather than raising, since the response has already started and an unhandled exception would silently truncate the file --- src/mavedb/routers/score_sets.py | 171 ++++++++++++----- src/mavedb/routers/variants.py | 141 +++++++++++++- tests/routers/test_score_set.py | 196 ++++++++++++++++++-- tests/routers/test_variant.py | 305 +++++++++++++++++++++++++++++++ 4 files changed, 755 insertions(+), 58 deletions(-) create mode 100644 tests/routers/test_variant.py diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 1f3aeec29..2dc7aeeb0 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -53,8 +53,16 @@ from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration -from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN -from mavedb.lib.score_set_csv import get_score_set_variants_as_csv, variants_to_csv_rows +from mavedb.lib.csv.deprecated_params import ( + DROP_NA_COLUMNS_DESCRIPTION, + INCLUDE_CUSTOM_COLUMNS_DESCRIPTION, + INCLUDE_POST_MAPPED_HGVS_DESCRIPTION, + resolve_deprecated_csv_params, +) +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr +from mavedb.view_models.csv_namespace import AvailableCsvNamespace +from mavedb.lib.csv.columns import variants_to_csv_rows +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.score_sets import ( csv_data_to_df, fetch_score_set_search_filter_options, @@ -908,6 +916,57 @@ async def show_score_set( return _score_set_response(item, principal) +@router.get( + "/score-sets/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this score set has data for", +) +def get_score_set_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + List the CSV column namespaces this score set has data for, labeled and grouped for a picker. + + Each entry's `namespace` is a value accepted by the `namespaces` parameter of the CSV endpoints. + Deliberately a separate request rather than a field on the score set: it costs several queries and is + only needed when a user opens a download dialog, so it should not sit on the score-set page's + critical path. + + Parameters + __________ + urn : str + The URN of the score set to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() + if not score_set: + logger.info(msg="Could not fetch CSV namespaces; No such score set exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found") + + assert_permission(user_data, score_set, Action.READ) + + return available_score_set_csv_namespaces( + db, + score_set, + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + ) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, @@ -927,17 +986,18 @@ def get_score_set_variants_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Maximum number of variants to return"), - namespaces: List[str] = Query( + namespaces: List[CsvNamespaceStr] = Query( default=["scores"], - description=( - 'One or more data types to include: "scores", "counts", "vep", "gnomad", "clingen", ' - 'and/or ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), + description=CSV_NAMESPACES_PARAM_DESCRIPTION, + ), + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), + include_post_mapped_hgvs: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_POST_MAPPED_HGVS_DESCRIPTION + ), + include_custom_columns: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_CUSTOM_COLUMNS_DESCRIPTION ), - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = None, - include_post_mapped_hgvs: Optional[bool] = None, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -957,11 +1017,19 @@ def get_score_set_variants_csv( The maximum number of variants to return. If None, returns all variants. namespaces: List[str] The namespaces of all columns except for accession, hgvs_nt, hgvs_pro, and hgvs_splice. - Supported values: "scores", "counts", "vep", "gnomad", "clingen", and ClinVar-versioned - namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" for January 2024). - Multiple ClinVar namespaces with different YEAR_MONTH values may be requested simultaneously. + Supported values: "scores" (the required score column), "scores_custom" (the investigator's + remaining score columns, emitted under the "scores" prefix), "counts", "mavedb", "vep", "gnomad", + "clingen", "score_set", and ClinVar- and calibration-parameterized namespaces. Multiple ClinVar + and calibration namespaces may be requested simultaneously. + drop_unused_hgvs_columns : bool, optional + Whether to omit the HGVS coordinate columns this score set does not use, e.g. hgvs_nt for a + protein-only score set. Defaults to False. drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. + Deprecated spelling of drop_unused_hgvs_columns, accepted for one release. + include_post_mapped_hgvs : bool, optional + Deprecated: equivalent to requesting the "mavedb" namespace. Accepted for one release. + include_custom_columns : bool, optional + Deprecated: equivalent to requesting the "scores_custom" namespace. Accepted for one release. db : Session The database session to use. user_data : Optional[UserData] @@ -972,13 +1040,23 @@ def get_score_set_variants_csv( str The CSV string containing the variant data. """ + deprecated = resolve_deprecated_csv_params( + namespaces=namespaces, + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + drop_na_columns=drop_na_columns, + include_post_mapped_hgvs=include_post_mapped_hgvs, + include_custom_columns=include_custom_columns, + ) + namespaces = deprecated.namespaces + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, "resource_property": "scores", "start": start, "limit": limit, - "drop_na_columns": drop_na_columns, + "drop_unused_hgvs_columns": drop_unused_hgvs_columns, } ) @@ -989,21 +1067,6 @@ def get_score_set_variants_csv( logger.info(msg="Could not fetch scores with non-positive limit.", extra=logging_context()) raise HTTPException(status_code=422, detail="Limit must be positive") - _VALID_STATIC_NAMESPACES = {"scores", "counts", "vep", "gnomad", "clingen"} - invalid_namespaces = [ - ns for ns in namespaces if ns not in _VALID_STATIC_NAMESPACES and not CLINVAR_NS_PATTERN.match(ns) - ] - if invalid_namespaces: - raise HTTPException( - status_code=422, - detail=( - f"Invalid namespace(s): {invalid_namespaces}. " - 'Each namespace must be one of "scores", "counts", "vep", "gnomad", "clingen", ' - 'or a ClinVar-versioned namespace of the form "clinvar.YEAR_MM" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), - ) - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: logger.info(msg="Could not fetch the requested scores; No such score set exists.", extra=logging_context()) @@ -1018,11 +1081,12 @@ def get_score_set_variants_csv( True, start, limit, - drop_na_columns, - include_custom_columns, - include_post_mapped_hgvs, + drop_unused_hgvs_columns, + # Asked separately from the score set: a private calibration is readable only by its owner, + # investigator contributors, or an admin, whoever can read the score set. + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, ) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1044,7 +1108,8 @@ def get_score_set_scores_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1056,6 +1121,11 @@ def get_score_set_scores_csv( /score-sets/{urn}/scores?start=0&limit=100 /score-sets/{urn}/scores?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1079,8 +1149,12 @@ def get_score_set_scores_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + # Both score namespaces: this endpoint has always returned every score column the investigator + # uploaded, and `scores` alone is now just the required one. + csv_str = get_score_set_variants_as_csv( + db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns + ) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1102,7 +1176,8 @@ async def get_score_set_counts_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1114,6 +1189,11 @@ async def get_score_set_counts_csv( /score-sets/{urn}/counts?start=0&limit=100 /score-sets/{urn}/counts?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1137,8 +1217,8 @@ async def get_score_set_counts_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1209,6 +1289,15 @@ def _stream_generated_annotations(mapped_variants, annotation_function): except MappingDataDoesntExistException: logger.debug(f"Mapping data does not exist for variant {mv.variant.urn}.") annotation = None + except Exception: + # Raising here would end the body mid-stream. The 200 and its headers went out with the first + # chunk, so the client has no way to be told and simply receives a short file. Report the + # variant as unannotated and keep going, so one bad variant cannot truncate a whole download. + logger.exception( + f"Failed to annotate variant {mv.variant.urn}; streaming it as unannotated.", + extra=logging_context(), + ) + annotation = None # Send pure result data (no wrapper) result = { diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index c195f9030..d9515097c 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,9 +1,11 @@ import itertools import logging import re +from typing import Any, List, Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from fastapi.exceptions import HTTPException +from fastapi.responses import StreamingResponse from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session, joinedload @@ -11,10 +13,12 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData +from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -25,6 +29,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) +from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.variant import ( ClingenAlleleIdVariantLookupResponse, ClingenAlleleIdVariantLookupsRequest, @@ -460,3 +465,137 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User assert_permission(user_data, variant.score_set, Action.READ) return variant + + +@router.get( + "/variants/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this variant has data for", +) +def get_variant_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + List the CSV column namespaces this variant has data for, labeled and grouped for a picker. + + Widens over the variant's equivalent measurements the same way the CSV does, so a calibration + belonging to another score set that also measured this allele is offered here too. + + Parameters + __________ + urn : str + The URN of the variant to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + if not variant: + logger.info(msg="Could not fetch CSV namespaces; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + return available_variant_csv_namespaces( + db, + urn, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + ) + + +@router.get( + "/variants/{urn}/csv", + status_code=200, + responses={ + 200: { + "content": {"text/csv": {}}, + "description": ( + "Variant data in CSV format, one row per measurement of the variant's allele. Columns" + " cover identity, mapped coordinates, the measured score, external annotations, and each" + " requested calibration's functional and ACMG interpretation." + ), + }, + **BASE_400_RESPONSE, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Get variant data in CSV format", +) +def get_variant_csv_data( + *, + urn: str, + namespaces: Optional[List[CsvNamespaceStr]] = Query(default=None, description=CSV_NAMESPACES_PARAM_DESCRIPTION), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + Return tabular data for a single variant, identified by URN, in CSV format. + + Where the variant-level annotation endpoints return nested VA-Spec objects, this flattens the same + interpretation into columns a clinical information system can consume: ACMG criteria, evidence + strengths, and evidence outcome codes alongside the measurement they were derived from. + + A row is emitted for every current measurement of the variant's ClinGen allele, so a variant assayed + in several score sets yields several rows. The requested variant is always first. + + Parameters + __________ + urn : str + The URN of the variant to fetch. + namespaces : Optional[List[str]] + The groups of columns to include. When omitted, the response includes the fixed groups plus one + namespace per calibration eligible to annotate these measurements and the most recent ClinVar + release covering them. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + Any + StreamingResponse containing the CSV data. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv", "namespaces": namespaces}) + + try: + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + # Only measurements the requester may read are emitted. The predicate runs against the score sets + # reached by the widening, keeping the permission check proportional to the result. + # A calibration's READ permission is stricter than its score set's, so it is asked separately: being + # able to read the measurement does not entitle a caller to a private calibration's interpretation. + csv_str = get_variant_csv( + db, + urn, + namespaces=namespaces, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + ) + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{urn}.csv"'}, + ) diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index b7ecb514d..7fb80c12c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -3428,7 +3428,7 @@ def test_download_variants_data_file( worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_na_columns=true&include_post_mapped_hgvs=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_unused_hgvs_columns=true&namespaces=scores&namespaces=mavedb" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3477,7 +3477,7 @@ def test_download_scores_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3499,7 +3499,7 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3510,6 +3510,144 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d assert "hgvs_splice" not in columns +# Deprecated query-parameter aliases. Galaxy and other external tooling call these endpoints, so the old +# names keep working for a release rather than being silently ignored. +def test_deprecated_drop_na_columns_still_drops_unused_hgvs_columns( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + for path in ("variants/data?namespaces=scores&", "scores?", "counts?"): + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}drop_na_columns=true") + + assert response.status_code == 200, path + columns = response.text.split("\n")[0].split(",") + assert "hgvs_splice" not in columns, path + + +def test_deprecated_include_post_mapped_hgvs_adds_the_mavedb_namespace( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + columns = response.text.split("\n")[0].split(",") + # Additive, as the flag always was: the requested namespace survives alongside it. + assert "scores.score" in columns + assert "mavedb.post_mapped_hgvs_g" in columns + + +def test_deprecated_include_custom_columns_adds_the_scores_custom_namespace( + session, data_provider, client, setup_router_db, data_files +): + """The flag now appends a namespace, and its columns keep the `scores.` prefix they always had.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with_flag = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_custom_columns=true" + ) + with_namespace = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=scores_custom" + ) + + assert with_flag.status_code == 200 + assert with_namespace.status_code == 200 + assert with_flag.text.split("\n")[0] == with_namespace.text.split("\n")[0] + assert with_flag.headers["Deprecation"] == "true" + assert "include_custom_columns is deprecated" in with_flag.headers["Warning"] + # No column is emitted under a `scores_custom.` prefix; the namespace is a request token only. + assert "scores_custom." not in with_flag.text + + +def test_current_parameter_name_wins_over_its_deprecated_spelling( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=false&drop_na_columns=true" + ) + + assert response.status_code == 200 + assert "hgvs_splice" in response.text.split("\n")[0].split(",") + + +def test_deprecated_request_answers_with_deprecation_headers( + session, data_provider, client, setup_router_db, data_files +): + """The consumers here are scripts, not people reading our logs, so the response has to say so.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + "?namespaces=scores&drop_na_columns=true&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + assert response.headers["deprecation"] == "true" + warning = response.headers["warning"] + assert "drop_na_columns is deprecated, use drop_unused_hgvs_columns" in warning + assert "include_post_mapped_hgvs is deprecated, use namespaces=mavedb" in warning + + +def test_current_request_carries_no_deprecation_headers(session, data_provider, client, setup_router_db, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" + ) + + assert response.status_code == 200 + assert "deprecation" not in response.headers + assert "warning" not in response.headers + + +def test_deprecated_parameters_are_marked_deprecated_in_the_openapi_schema(client): + """Anyone reading the docs or generating a client should see the deprecation without sending a request.""" + schema = client.app.openapi() + + def parameter(path: str, name: str): + return next(p for p in schema["paths"][path]["get"]["parameters"] if p["name"] == name) + + for path, name in ( + ("/api/v1/score-sets/{urn}/variants/data", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/variants/data", "include_post_mapped_hgvs"), + ("/api/v1/score-sets/{urn}/scores", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/counts", "drop_na_columns"), + ): + assert parameter(path, name)["deprecated"] is True, f"{name} on {path}" + assert "deprecated" in parameter(path, name)["description"].lower(), f"{name} on {path}" + + # Namespace variant CSV export tests. def test_download_scores_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) @@ -3522,7 +3660,7 @@ def test_download_scores_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3545,7 +3683,7 @@ def test_download_counts_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3569,7 +3707,7 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r worker_queue.assert_called_once() download_scores_and_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_scores_and_counts_csv_response.status_code == 200 download_scores_and_counts_csv = download_scores_and_counts_csv_response.text @@ -3604,7 +3742,7 @@ def test_download_scores_counts_and_post_mapped_variants_file( worker_queue.assert_called_once() download_multiple_data_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&include_custom_columns=true&include_post_mapped_hgvs=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&namespaces=mavedb&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_multiple_data_csv_response.status_code == 200 download_multiple_data_csv = download_multiple_data_csv_response.text @@ -3643,7 +3781,7 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3672,7 +3810,7 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3694,13 +3832,39 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames +def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( + session, data_provider, client, setup_router_db, data_files +): + """A variant linked only to a gnomAD record of another version must still appear, with an NA frequency. + + The version filter belongs in the join's ON clause; in a WHERE it silently drops the variant row. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # The seeded gnomAD variant's version deliberately differs from the configured export version. + link_gnomad_variants_to_mapped_variants(session, score_set) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad") + assert response.status_code == 200 + + rows = list(csv.DictReader(StringIO(response.text))) + assert len(rows) == 3, "every variant must be present regardless of linked gnomAD versions" + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows) + + def test_download_clingen_and_vep_file_in_variant_data_path( session, data_provider, client, setup_router_db, data_files ): @@ -3722,7 +3886,7 @@ def test_download_clingen_and_vep_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3754,7 +3918,7 @@ def test_download_clingen_and_scores_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3790,7 +3954,7 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3824,7 +3988,7 @@ def test_download_clinvar_namespace_with_no_matching_version( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3854,7 +4018,7 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_na_columns=false" + f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -4066,7 +4230,7 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( publish_score_set = publish_score_set_response.json() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py new file mode 100644 index 000000000..14de3a9f6 --- /dev/null +++ b/tests/routers/test_variant.py @@ -0,0 +1,305 @@ +# ruff: noqa: E402 + +import csv +from io import StringIO +from unittest.mock import patch +from urllib.parse import quote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from sqlalchemy import select + +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + link_clinvar_control_to_mapped_variant, + publish_score_set, +) + + +def _first_variant_urn(session, score_set_urn): + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + return score_set.variants[0].urn + + +def _csv_path(variant_urn): + """Variant URNs contain a '#', which must be percent-encoded or it is read as a URL fragment.""" + return f"/api/v1/variants/{quote(variant_urn, safe='')}/csv" + + +def _published_score_set_with_mapped_variants(client, session, data_provider, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + return published + + +class TestGetVariantCsv: + def test_returns_csv_attachment(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/csv") + assert response.headers["content-disposition"] == f'attachment; filename="{variant_urn}.csv"' + + def test_reports_the_requested_variant(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + rows = list(csv.DictReader(StringIO(response.text))) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant_urn + assert rows[0]["score_set.score_set_urn"] == published["urn"] + assert rows[0]["relationship.match_type"] == "exact" + + def test_namespaces_restrict_the_columns(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces=scores") + + assert response.status_code == 200 + header = response.text.splitlines()[0] + assert "scores.score" in header + assert "gnomad.gnomad_af" not in header + assert "relationship.match_type" not in header + + def test_clinvar_namespace_is_labeled_with_its_release( + self, session, data_provider, client, setup_router_db, data_files + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + # The seeded ClinVar control is release 11_2024. + assert "clinvar.2024_11.clinical_significance" in response.text.splitlines()[0] + + @pytest.mark.parametrize( + "namespace", + ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.not-a-urn"], + ) + def test_invalid_namespace_is_rejected( + self, session, data_provider, client, setup_router_db, data_files, namespace + ): + """FastAPI validates the namespace vocabulary from the parameter type, before the handler runs.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 422 + errors = response.json()["detail"] + assert any(error["input"] == namespace for error in errors) + + @pytest.mark.parametrize("namespace", ["scores", "clinvar.2024_01"]) + def test_valid_namespace_is_accepted(self, session, data_provider, client, setup_router_db, data_files, namespace): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 200 + + def test_namespace_vocabulary_is_published_to_openapi(self, client): + """The generated schema is what the frontend builds its namespace selector from.""" + schema = client.app.openapi()["paths"]["/api/v1/variants/{urn}/csv"]["get"] + namespaces_param = next(param for param in schema["parameters"] if param["name"] == "namespaces") + + item_schema = next( + option["items"] for option in namespaces_param["schema"]["anyOf"] if option.get("type") == "array" + ) + published = {value for option in item_schema["anyOf"] if "enum" in option for value in option["enum"]} + patterns = [option["pattern"] for option in item_schema["anyOf"] if "pattern" in option] + + assert { + "scores", + "scores_custom", + "counts", + "mavedb", + "vep", + "gnomad", + "clingen", + "score_set", + "relationship", + } == published + assert any("clinvar" in pattern for pattern in patterns) + assert any("calibration" in pattern for pattern in patterns) + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(_csv_path("urn:mavedb:00000000-a-1#1")) + + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + def test_private_score_set_is_not_readable_by_other_users( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = _first_variant_urn(session, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 404 + + +class TestPrivateCalibrationsAreNotServedOverHttp: + """The lib tests cover the gating logic; this covers the wiring that reaches it. + + The predicate is hand-threaded through four signatures, so the endpoint is the contract worth pinning: + a caller who may read the score set but not the calibration must get NA, even naming the URN outright. + """ + + def _private_calibration(self, session, score_set_urn): + from mavedb.models.score_calibration import ScoreCalibration + + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn="urn:mavedb:calibration-99999999-9999-9999-9999-999999999999", + title="Unpublished Calibration", + baseline_score=0.0, + research_use_only=False, + primary=False, + private=True, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + session.add(calibration) + session.commit() + return calibration.urn + + def test_another_user_naming_the_urn_gets_no_interpretation( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + namespace = f"calibration.{calibration_urn}" + + with DependencyOverrider(extra_user_app_overrides): + response = client.get( + f"/api/v1/score-sets/{published['urn']}/variants/data?namespaces=scores&namespaces={quote(namespace)}" + ) + + assert response.status_code == 200 + rows = list(csv.DictReader(StringIO(response.text))) + assert rows, "expected variant rows" + assert all(row[f"{namespace}.title"] == "NA" for row in rows) + + def test_another_user_is_not_offered_it_by_discovery( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + assert f"calibration.{calibration_urn}" not in [entry["namespace"] for entry in response.json()] + + +class TestCsvNamespaceDiscovery: + """The discovery endpoints advertise what a namespace picker should offer.""" + + def test_score_set_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + entries = response.json() + by_namespace = {entry["namespace"]: entry for entry in entries} + assert {"scores", "score_set", "vep", "gnomad", "clingen"} <= set(by_namespace) + assert "relationship" not in by_namespace + assert by_namespace["gnomad"]["label"] == "gnomAD allele frequency" + assert by_namespace["gnomad"]["group"] == "annotation" + # Every entry is renderable without the client inventing labels. + assert all(entry["label"] and entry["group"] for entry in entries) + + def test_score_set_detail_response_is_unchanged(self, session, data_provider, client, setup_router_db, data_files): + """Discovery is its own request, so it must not appear on the score-set page's critical path.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}") + + assert response.status_code == 200 + assert "availableCsvNamespaces" not in response.json() + + def test_variant_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces") + + assert response.status_code == 200 + by_namespace = {entry["namespace"]: entry for entry in response.json()} + # The variant CSV does emit relationship columns, unlike the score-set CSV. + assert "relationship" in by_namespace + assert by_namespace["relationship"]["group"] == "provenance" + + def test_advertised_namespaces_are_accepted_by_the_csv_endpoints( + self, session, data_provider, client, setup_router_db, data_files + ): + """Discovery and validation must agree, or the picker offers options that 422.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + entries = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces").json() + namespaces = [entry["namespace"] for entry in entries] + assert "clinvar.2024_11" in namespaces + + query = "&".join(f"namespaces={quote(ns, safe='')}" for ns in namespaces) + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants/data?{query}") + assert response.status_code == 200 + + variant_urn = _first_variant_urn(session, published["urn"]) + variant_entries = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces").json() + variant_query = "&".join(f"namespaces={quote(entry['namespace'], safe='')}" for entry in variant_entries) + response = client.get(f"{_csv_path(variant_urn)}?{variant_query}") + assert response.status_code == 200 + + def test_unknown_score_set_returns_404(self, client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/csv-namespaces") + + assert response.status_code == 404 + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(f"/api/v1/variants/{quote('urn:mavedb:00000000-a-1#1', safe='')}/csv-namespaces") + + assert response.status_code == 404 From 69bc2ea59737b633117980afeb0003862095f1b5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 10:37:47 -0700 Subject: [PATCH 4/9] feat(export): derive public dump namespaces from CSV discovery Replace the hand-maintained list of ClinVar release namespaces in export_public_data.py with annotation_export_namespaces(), which asks available_score_set_csv_namespaces() what the score set actually has and subtracts the score/count/identity groups already covered by their own files in the dump. The previous list named releases one by one, so it silently emitted all-NA columns for a release never ingested and needed a code change for every new one. Discovery-derived namespaces also add score calibration interpretations to the annotations CSV for the first time, since those are now part of what discovery reports. Update scripts/resources/README.md to document the calibration column group and clarify that the ClinVar and calibration groups vary by score set and should be read from the header rather than assumed fixed. --- src/mavedb/scripts/export_public_data.py | 58 ++++++++++++++++-------- src/mavedb/scripts/resources/README.md | 39 +++++++++++++++- 2 files changed, 76 insertions(+), 21 deletions(-) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index b390aadeb..aaaafc831 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -26,9 +26,13 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.score_set import ( + available_score_set_csv_namespaces, + get_score_set_variants_as_csv, +) from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_set_csv import get_score_set_variants_as_csv from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet @@ -46,6 +50,39 @@ T = TypeVar("T") +def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: + """The namespaces the public annotations CSV should carry for this score set. + + Asks discovery what the score set actually has rather than naming groups by hand. The previous + hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases + never ingested, needed a code change for every new release, and was fragile to schema changes. + + The archive carries everything MaveDB holds about the score set, so this takes what discovery found + and subtracts from it rather than opting groups in. + + In particular it does not filter on `selected_by_default`. That flag answers "what should a download + dialog open on", which is a question about attention rather than about what exists, and the reasons a + group opens unchecked are not interchangeable. An archive is about completeness, not about what a user + should be nudged to look at first. + + Subtractions: + + - Every score and count group, and the score set's own identity: scores and counts get their own + files, and the URN is in the filename, so repeating either would be noise. + """ + excluded = { + CsvNamespace.SCORES, + CsvNamespace.SCORES_CUSTOM, + CsvNamespace.COUNTS, + CsvNamespace.SCORE_SET, + } + return [ + entry.namespace + for entry in available_score_set_csv_namespaces(db, score_set) + if entry.namespace not in excluded + ] + + def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: return chain.from_iterable(map(f, items)) @@ -205,24 +242,7 @@ def export_public_data(db: Session): csv_str = get_score_set_variants_as_csv( db, score_set, - [ - "vep", - "gnomad", - "clingen", - "clinvar.2015_02", - "clinvar.2016_01", - "clinvar.2017_01", - "clinvar.2018_01", - "clinvar.2019_01", - "clinvar.2020_01", - "clinvar.2021_01", - "clinvar.2022_01", - "clinvar.2023_01", - "clinvar.2024_01", - "clinvar.2025_01", - "clinvar.2026_01", - ], - include_post_mapped_hgvs=True, + annotation_export_namespaces(db, score_set), namespaced=True, ) zipfile.writestr(f"csv/{csv_filename_base}.annotations.csv", csv_str) diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 31ec4e26c..30d244a62 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -35,7 +35,7 @@ mavedb-dump.YYYYMMDDHHMMSS.zip ├── csv/ │ ├── {urn}.scores.csv # Variant effect scores (all score sets) │ ├── {urn}.counts.csv # Variant counts (score sets with count data only) -│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, and ClinGen +│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, ClinGen and ClinVar, plus score calibration interpretations │ # (score sets that have completed mapping only) ├── mapped/ │ └── {urn}.mapped-variants.json # Mapped variant data including VRS alleles and HGVS @@ -114,7 +114,9 @@ present for score sets that have count data. The count column names are listed i Variant annotation data from external databases, joined with post-mapped HGVS and VRS identifiers produced by the MaveDB variant mapping pipeline. **Only present for score sets that have completed -the MaveDB mapping pipeline.** Exact columns: +the MaveDB mapping pipeline.** + +Columns are grouped by a namespace prefix. The groups below always appear: | Column | Description | |--------|-------------| @@ -131,6 +133,39 @@ the MaveDB mapping pipeline.** Exact columns: | `gnomad.gnomad_af` | gnomAD v4.1 allele frequency | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | +Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. +Read the header rather than assuming a fixed column set. + +**ClinVar** — one pair of columns per ingested release, prefixed `clinvar.YEAR_MONTH`. A score set with +records from the January 2024 release carries `clinvar.2024_01.clinical_significance` and +`clinvar.2024_01.clinical_review_status`. + +This file carries **every** release MaveDB holds for the score set, not just the most recent one, so a +change in ClinVar's assessment over time can be read off a single file. + +**Score calibrations** — one group per calibration, prefixed `calibration.`, giving +that calibration's interpretation of each variant: + +| Column suffix | Description | +|---------------|-------------| +| `title` | Human-readable name of the calibration | +| `research_use_only` | Always `False` here; research-use-only calibrations are excluded from this dump | +| `functional_classification` | `normal`, `abnormal`, or `indeterminate` | +| `acmg_criterion` | ACMG 2015 criterion evaluated, e.g. `PS3` or `BS3` | +| `acmg_evidence_strength` | Strength the criterion was met at, e.g. `MODERATE`. `NA` when not met | +| `acmg_evidence_outcome_code` | ACMG evidence outcome code, e.g. `PS3_moderate`, `PS3` (strong), `BS3_not_met` | +| `pathogenicity_classification` | `PATHOGENIC`, `BENIGN`, or `UNCERTAIN_SIGNIFICANCE` | + +Every calibration MaveDB holds for the score set gets a group, including one that defines no score +ranges. Such a group carries `title` and +`research_use_only` with `NA` in every interpretation column: the calibration exists and was consulted, +and it has no classification to give. That is different from a calibration whose ranges simply do not +contain a particular variant, which reports `UNCERTAIN_SIGNIFICANCE` and `PS3_not_met`. + +`acmg_evidence_strength` uses MaveDB's own scale, which includes `MODERATE_PLUS` — an intermediate +strength that the GA4GH VA-Spec has no equivalent for. The same variant's record in `va/{urn}.va.ndjson` +therefore reports `moderate` where this file reports `MODERATE_PLUS`. + Variants that could not be mapped, or for which a specific annotation is unavailable, will have `NA` in the corresponding column. For multi-allelic variants (haplotypes), `mavedb.*` HGVS columns will be `NA` because a single combined HGVS string cannot currently be derived. This may be updated in From adfd63059e1fd08ea5b75debdda343cce9a9c02a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 16:09:32 -0700 Subject: [PATCH 5/9] refactor(csv): filter calibrations through ScoreCalibrationViewer --- src/mavedb/lib/csv/annotations.py | 17 ++++++++--------- src/mavedb/lib/csv/entries.py | 24 +++++++++++------------- src/mavedb/lib/csv/score_set.py | 15 +++++++-------- src/mavedb/lib/csv/variant.py | 21 +++++++++------------ src/mavedb/routers/score_sets.py | 6 ++++-- src/mavedb/routers/variants.py | 9 +++++++-- tests/lib/csv/test_variant.py | 14 +++++++++++--- 7 files changed, 57 insertions(+), 49 deletions(-) diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py index 5f5736e8e..3469d6d64 100644 --- a/src/mavedb/lib/csv/annotations.py +++ b/src/mavedb/lib/csv/annotations.py @@ -3,13 +3,14 @@ Shared by both exports, and kept out of ``columns`` because filling these cells needs the database. """ -from typing import Callable, Optional, Sequence +from typing import Optional, Sequence from sqlalchemy import select from sqlalchemy.orm import Session, selectinload from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation -from mavedb.lib.csv.entries import visible_calibrations +from mavedb.lib.csv.entries import calibration_viewer +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -22,13 +23,13 @@ def calibrations_for_namespaces( db: Session, calibration_namespaces: dict[str, str], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> dict[str, ScoreCalibration]: """Load the calibrations named by the requested namespaces, keyed by namespace. Looked up by the URN the caller named, not by what a score set offers: the namespace *is* the request. Which means this, not discovery, is the gate — naming a private calibration's URN directly must not - serve its interpretation, so *may_read_calibration* is applied here too. + serve its interpretation, so the viewer is applied here too. """ if not calibration_namespaces: return {} @@ -43,9 +44,7 @@ def calibrations_for_namespaces( ) ).all() - by_urn = { - str(calibration.urn): calibration for calibration in visible_calibrations(calibrations, may_read_calibration) - } + by_urn = {str(calibration.urn): calibration for calibration in calibration_viewer(viewer).visible(calibrations)} return {namespace: by_urn[urn] for namespace, urn in calibration_namespaces.items() if urn in by_urn} @@ -77,7 +76,7 @@ def annotations_for_rows( variants: Sequence[Variant], mappings: Sequence[Optional[MappedVariant]], calibration_namespaces: dict[str, str], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: """Flatten every row's interpretation under each requested calibration namespace. @@ -90,7 +89,7 @@ def annotations_for_rows( if not calibration_namespaces: return None - calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, may_read_calibration) + calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, viewer) # TODO(#372): non-null id fields membership = containing_classification_ids(db, [variant.id for variant in variants]) # type: ignore diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py index ea57687e4..ba38b7f60 100644 --- a/src/mavedb/lib/csv/entries.py +++ b/src/mavedb/lib/csv/entries.py @@ -6,7 +6,7 @@ """ from dataclasses import dataclass -from typing import Callable, Iterable, Optional, Sequence +from typing import Iterable, Optional, Sequence from sqlalchemy import and_, select from sqlalchemy.orm import Session @@ -21,6 +21,7 @@ clinvar_namespace_label, clinvar_namespace_sort_key, ) +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.clinical_control import ClinicalControl from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration @@ -82,22 +83,19 @@ def clinvar_namespace_entries(namespaces: Iterable[str]) -> list[AvailableCsvNam return entries -def visible_calibrations( - calibrations: Iterable[ScoreCalibration], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, -) -> list[ScoreCalibration]: - """Drop calibrations the caller may not read. +def calibration_viewer(viewer: Optional[ScoreCalibrationViewer]) -> ScoreCalibrationViewer: + """Resolve an omitted viewer to the anonymous one. A calibration carries its own ``private`` flag, and its READ permission is stricter than its score set's: a private one is readable only by its owner, by contributors when it is investigator-provided, - or by an admin. Reading the score set is not enough, so every path that names a calibration has to ask - separately. + or by an admin. Reading the score set is not enough, so every CSV path that names a calibration has to + ask separately. - Defaults to public-only, and treats an unset ``private`` as private. A caller that forgets to pass a - predicate therefore gets the subset anyone could see rather than everything. + This is the single place the CSV package decides what an absent viewer means, and it means the public + subset: a call site that forgets to thread one serves what anyone could already see rather than + everything. The rule itself lives in ``ScoreCalibrationViewer``, so it is never restated here. """ - permitted = may_read_calibration or (lambda calibration: calibration.private is False) - return [calibration for calibration in calibrations if permitted(calibration)] + return viewer if viewer is not None else ScoreCalibrationViewer() def calibration_can_annotate(calibration: ScoreCalibration) -> bool: @@ -105,7 +103,7 @@ def calibration_can_annotate(calibration: ScoreCalibration) -> bool: False for a calibration with no score ranges, whose every cell would be NA. Research-use-only standing is excluded from this question — it asks what a calibration *could* say, while who may see it is - ``visible_calibrations``' job. + ``ScoreCalibrationViewer``'s job. """ return any( score_calibration_may_be_used_for_annotation( diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py index 148cc0bd9..9aafad32d 100644 --- a/src/mavedb/lib/csv/score_set.py +++ b/src/mavedb/lib/csv/score_set.py @@ -1,6 +1,6 @@ """The score-set CSV export: every variant in one score set, and the columns it can offer.""" -from typing import Callable, List, Optional +from typing import List, Optional from sqlalchemy import and_, select from sqlalchemy.orm import Session, selectinload @@ -16,7 +16,7 @@ from mavedb.lib.csv.entries import ( AvailableCsvNamespaceEntry, calibration_namespace_entries, - visible_calibrations, + calibration_viewer, clinvar_namespace_entries, clinvar_release_namespaces, score_sets_have_current_mappings, @@ -25,6 +25,7 @@ from mavedb.lib.csv.fetch import fetch_variant_csv_data from mavedb.lib.csv.namespaces import CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet @@ -37,7 +38,7 @@ def get_score_set_variants_as_csv( start: Optional[int] = None, limit: Optional[int] = None, drop_unused_hgvs_columns_flag: Optional[bool] = None, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> str: """Get the variant data from a score set as a CSV string.""" assert type(score_set.dataset_columns) is dict @@ -61,9 +62,7 @@ def get_score_set_variants_as_csv( mappings=fetched.mappings, gnomad_data=fetched.gnomad_data, clinvar_data_by_ns=fetched.clinvar_per_variant, - annotations_by_ns=annotations_for_rows( - db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration - ), + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), ) rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) @@ -77,7 +76,7 @@ def get_score_set_variants_as_csv( def available_score_set_csv_namespaces( db: Session, score_set: ScoreSet, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. @@ -117,7 +116,7 @@ def available_score_set_csv_namespaces( ) .where(and_(ScoreCalibration.score_set_id == score_set.id, ScoreCalibration.urn.is_not(None))) ).all() - entries.extend(calibration_namespace_entries(visible_calibrations(calibrations, may_read_calibration))) + entries.extend(calibration_namespace_entries(calibration_viewer(viewer).visible(calibrations))) # `relationship` is absent by design: match_type describes a row's relation to a requested record, # which only the variant CSV has. diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py index 66cccdfa0..61e657e76 100644 --- a/src/mavedb/lib/csv/variant.py +++ b/src/mavedb/lib/csv/variant.py @@ -29,7 +29,7 @@ clinvar_release_namespaces, score_sets_have_current_mappings, static_namespace_entry, - visible_calibrations, + calibration_viewer, ) from mavedb.lib.csv.fetch import fetch_variant_csv_data from mavedb.lib.csv.namespaces import ( @@ -39,6 +39,7 @@ ) from mavedb.lib.mave.utils import NA_VALUE from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -192,7 +193,7 @@ def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional def _annotatable_calibration_namespaces( db: Session, score_set_ids: list[int], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> dict[str, ScoreCalibration]: """Map calibration namespace to calibration, for every calibration eligible to annotate these variants. @@ -216,7 +217,7 @@ def _annotatable_calibration_namespaces( ).all() namespaces: dict[str, ScoreCalibration] = {} - for calibration in visible_calibrations(calibrations, may_read_calibration): + for calibration in calibration_viewer(viewer).visible(calibrations): if not calibration.urn: continue @@ -234,7 +235,7 @@ def available_variant_csv_namespaces( db: Session, variant_urn: str, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. @@ -262,9 +263,7 @@ def available_variant_csv_namespaces( return ( base_entries - + calibration_namespace_entries( - _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration).values() - ) + + calibration_namespace_entries(_annotatable_calibration_namespaces(db, score_set_ids, viewer).values()) + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) ) @@ -274,7 +273,7 @@ def get_variant_csv( variant_urn: str, namespaces: Optional[list[str]] = None, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, na_rep: str = NA_VALUE, ) -> str: """Build the clinical CSV for a variant and its equivalent measurements. @@ -299,7 +298,7 @@ def get_variant_csv( mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) - calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration) + calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, viewer) if namespaces is None: clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) @@ -333,9 +332,7 @@ def get_variant_csv( mappings=fetched.mappings, gnomad_data=fetched.gnomad_data, clinvar_data_by_ns=fetched.clinvar_per_variant, - annotations_by_ns=annotations_for_rows( - db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration - ), + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), match_types=[EXACT_MATCH_TYPE] * len(fetched.variants), na_rep=na_rep, namespaced=True, diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 2dc7aeeb0..de0cda700 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -928,6 +928,7 @@ def get_score_set_csv_namespaces( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ List the CSV column namespaces this score set has data for, labeled and grouped for a picker. @@ -963,7 +964,7 @@ def get_score_set_csv_namespaces( return available_score_set_csv_namespaces( db, score_set, - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) @@ -1000,6 +1001,7 @@ def get_score_set_variants_csv( ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return tabular variant data from a score set, identified by URN, in CSV format. @@ -1084,7 +1086,7 @@ def get_score_set_variants_csv( drop_unused_hgvs_columns, # Asked separately from the score set: a private calibration is readable only by its owner, # investigator contributors, or an admin, whoever can read the score set. - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index d9515097c..bb76716c4 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -13,9 +13,12 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user +from mavedb.lib.authorization import get_principal from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv @@ -479,6 +482,7 @@ def get_variant_csv_namespaces( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ List the CSV column namespaces this variant has data for, labeled and grouped for a picker. @@ -513,7 +517,7 @@ def get_variant_csv_namespaces( db, urn, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) @@ -540,6 +544,7 @@ def get_variant_csv_data( namespaces: Optional[List[CsvNamespaceStr]] = Query(default=None, description=CSV_NAMESPACES_PARAM_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return tabular data for a single variant, identified by URN, in CSV format. @@ -592,7 +597,7 @@ def get_variant_csv_data( urn, namespaces=namespaces, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) return StreamingResponse( iter([csv_str]), diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 16d52dbc5..fd205f1cc 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -3,7 +3,7 @@ import csv import io from datetime import date -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -18,9 +18,12 @@ available_variant_csv_namespaces, get_variant_csv, ) +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.acmg_classification import ACMGClassification from mavedb.models.clinical_control import ClinicalControl from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.user_role import UserRole from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant @@ -1082,15 +1085,20 @@ def test_score_set_csv_withholds_it_too(self, session, private_calibration): assert all(row[f"{CALIBRATION_NS_1}.title"] == "NA" for row in rows) def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_mapped_variant, private_calibration): - """The predicate widens access; it must not be a blanket ban on private calibrations.""" + """Viewer-scoped emission: the viewer widens access, it is not a blanket ban on private calibrations. + + Uses a real entitled viewer rather than an always-true stand-in, so this exercises the same + ``ScoreCalibrationViewer`` rule the routers use. + """ variant = setup_lib_db_with_mapped_variant.variant + admin = Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) rows = _parse_csv( get_variant_csv( session, variant.urn, ["scores", CALIBRATION_NS_1], - may_read_calibration=lambda calibration: True, + viewer=admin.viewer_for(ScoreCalibrationViewer), ) ) From 1d1d36f09b181c3977671ec9dc949e48ef0d1171 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 12:26:56 -0700 Subject: [PATCH 6/9] test(csv): cover the deprecated-param layer and collapse redundant null tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The namespace refactor moved risk into the compatibility layer while the test mass stayed where it had always been, on the shared NA coercion. - Add test_deprecated_params.py. This layer is all that keeps a pre-namespace client working: FastAPI ignores unknown query parameters, so an unmapped old spelling would silently return different columns rather than erroring, and Galaxy calls these endpoints. It is pure and total, so it is specified here rather than reached incidentally through the routers — unit coverage goes from 47% to 100%. Pins the precedence rules (the current name wins, the boolean flags append rather than replace), namespace-append idempotency, the published token spellings, and the RFC 8594 response headers. - Collapse TestVariantToCsvRowNullHandling from 15 tests to 5. Eight of them asserted the value -> NA rule through different namespaces, but every namespace reaches it through the single _value_or_na call in variant_to_csv_row, and test_is_output_null already specifies that rule over 18 values — so those tests could not fail unless it failed too. What remains covers the layer above: building the per-row source a resolver is handed, where absent data has several distinct shapes. Score and count columns are one mechanism selected by RowSource, so they are parametrized together instead of hand-duplicated. Deliberately not added: per-namespace null tests, and registry invariants. test_specs.py already asserts every namespace has a spec, every declared column resolves, every resolver tolerates a None source, and that a namespace reading through a relationship declares the fetch it needs. --- tests/lib/csv/test_columns.py | 155 ++++++------------------ tests/lib/csv/test_deprecated_params.py | 141 +++++++++++++++++++++ 2 files changed, 180 insertions(+), 116 deletions(-) create mode 100644 tests/lib/csv/test_deprecated_params.py diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index 719794bee..c00da889c 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -33,143 +33,66 @@ def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=No class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" + """How a row reports absent data. - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) + The value -> NA rule itself is specified by ``test_is_output_null``; every namespace reaches it through + the single ``_value_or_na`` call in ``variant_to_csv_row``, so it is not re-asserted per namespace here. + What these cover is the layer above it: building the per-row source a resolver is handed, which is + where "no data" has several distinct shapes. + """ - assert row["count1"] == "NA" + # The score and count namespaces are one mechanism selected by RowSource, so they are covered together. + DATA_NAMESPACES = [("scores", "score_data", "score"), ("counts", "count_data", "count1")] - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} + @pytest.mark.parametrize("namespace, data_key, column", DATA_NAMESPACES) + @pytest.mark.parametrize( + "build_data", + [lambda data_key: None, lambda data_key: {}, lambda data_key: {data_key: {}}], + ids=["no_data_at_all", "no_entry_for_this_namespace", "entry_present_but_column_missing"], + ) + def test_dynamic_columns_survive_every_shape_of_absent_data(self, namespace, data_key, column, build_data): + """``variant.data`` may be missing, lack this namespace's entry, or lack the column within it.""" + variant = MockVariant(data=build_data(data_key)) - row = variant_to_csv_row(variant, columns) + row = variant_to_csv_row(variant, {namespace: [column]}) - assert row["score"] == "1.5" + assert row[column] == "NA" - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} + @pytest.mark.parametrize("namespace, data_key, column", DATA_NAMESPACES) + def test_a_present_value_is_stringified(self, namespace, data_key, column): + """The non-null half of ``_value_or_na``, which nothing else asserts directly.""" + variant = MockVariant(data={data_key: {column: 1.5}}) - row = variant_to_csv_row(variant, columns) + row = variant_to_csv_row(variant, {namespace: [column]}) - assert row["count1"] == "42" + assert row[column] == "1.5" - def test_score_data_with_custom_na_rep(self): + def test_na_rep_is_configurable(self): variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - row = variant_to_csv_row(variant, columns, na_rep="N/A") + row = variant_to_csv_row(variant, {"scores": ["score"]}, na_rep="N/A") assert row["score"] == "N/A" - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): + def test_an_absent_column_does_not_disturb_its_neighbours(self): + """One missing datum must not drop a key or corrupt another column in the same row.""" variant = MockVariant( hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", + hgvs_pro=None, data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } + columns = {"core": ["hgvs_nt", "hgvs_pro"], "scores": ["score", "se"], "counts": ["count1", "count2"]} row = variant_to_csv_row(variant, columns) - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" + assert row == { + "hgvs_nt": "g.1A>G", + "hgvs_pro": "NA", + "score": "NA", + "se": "0.1", + "count1": "NA", + "count2": "5", + } # --------------------------------------------------------------------------- diff --git a/tests/lib/csv/test_deprecated_params.py b/tests/lib/csv/test_deprecated_params.py new file mode 100644 index 000000000..e5c032463 --- /dev/null +++ b/tests/lib/csv/test_deprecated_params.py @@ -0,0 +1,141 @@ +"""The compatibility layer keeping pre-namespace CSV clients working. + +This is the only thing standing between a client that predates the namespace vocabulary and silently +different output: FastAPI ignores unknown query parameters, so an unmapped old spelling would not error, +it would just stop returning columns. Galaxy calls these endpoints, so the translation is pinned here +rather than left to the router tests. +""" + +import pytest + +from mavedb.lib.csv.deprecated_params import resolve_deprecated_csv_params +from mavedb.lib.csv.namespaces import CsvNamespace + + +class TestNoDeprecatedParams: + """A current request must pass through untouched and announce nothing.""" + + def test_current_params_are_returned_unchanged(self): + resolved = resolve_deprecated_csv_params(namespaces=["scores", "gnomad"], drop_unused_hgvs_columns=True) + + assert resolved.namespaces == ["scores", "gnomad"] + assert resolved.drop_unused_hgvs_columns is True + assert resolved.deprecations == {} + assert resolved.response_headers == {} + + def test_no_params_at_all_is_not_an_error(self): + resolved = resolve_deprecated_csv_params() + + assert resolved.namespaces == [] + assert resolved.drop_unused_hgvs_columns is None + assert resolved.response_headers == {} + + def test_the_caller_s_namespace_list_is_not_mutated(self): + """The router hands in its own list; appending to it in place would leak across requests.""" + requested = ["scores"] + + resolve_deprecated_csv_params(namespaces=requested, include_post_mapped_hgvs=True) + + assert requested == ["scores"] + + +class TestDropNaColumns: + """``drop_na_columns`` -> ``drop_unused_hgvs_columns``: a rename, so the value carries over as-is.""" + + @pytest.mark.parametrize("value", [True, False]) + def test_the_deprecated_value_is_adopted_including_an_explicit_false(self, value): + """False is a real request to keep the columns, distinct from the parameter being absent.""" + resolved = resolve_deprecated_csv_params(drop_na_columns=value) + + assert resolved.drop_unused_hgvs_columns is value + assert resolved.deprecations == {"drop_na_columns": "drop_unused_hgvs_columns"} + + @pytest.mark.parametrize("current, deprecated", [(True, False), (False, True)]) + def test_the_current_name_wins_when_both_are_given(self, current, deprecated): + resolved = resolve_deprecated_csv_params(drop_unused_hgvs_columns=current, drop_na_columns=deprecated) + + assert resolved.drop_unused_hgvs_columns is current + + def test_both_given_announces_nothing(self): + """Current behaviour: the deprecated value was ignored, so no warning is raised about it.""" + resolved = resolve_deprecated_csv_params(drop_unused_hgvs_columns=True, drop_na_columns=False) + + assert resolved.deprecations == {} + assert resolved.response_headers == {} + + +class TestBooleanFlagsBecomeNamespaces: + """The two flags append a namespace; they were always additive to whatever columns were requested.""" + + @pytest.mark.parametrize( + "flag, expected_namespace, replacement", + [ + ("include_post_mapped_hgvs", CsvNamespace.REFERENCE_HGVS, "namespaces=mavedb"), + ("include_custom_columns", CsvNamespace.SCORES_CUSTOM, "namespaces=scores_custom"), + ], + ) + def test_a_set_flag_appends_its_namespace_and_keeps_the_requested_ones(self, flag, expected_namespace, replacement): + resolved = resolve_deprecated_csv_params(namespaces=["scores"], **{flag: True}) + + assert resolved.namespaces == ["scores", expected_namespace] + assert resolved.deprecations == {flag: replacement} + + @pytest.mark.parametrize("flag", ["include_post_mapped_hgvs", "include_custom_columns"]) + @pytest.mark.parametrize("value", [False, None]) + def test_an_unset_flag_is_a_no_op(self, flag, value): + resolved = resolve_deprecated_csv_params(namespaces=["scores"], **{flag: value}) + + assert resolved.namespaces == ["scores"] + assert resolved.deprecations == {} + + @pytest.mark.parametrize( + "flag, namespace", + [ + ("include_post_mapped_hgvs", CsvNamespace.REFERENCE_HGVS), + ("include_custom_columns", CsvNamespace.SCORES_CUSTOM), + ], + ) + def test_a_namespace_already_requested_is_not_appended_twice(self, flag, namespace): + """A duplicate namespace would emit its columns twice and collide in the header.""" + resolved = resolve_deprecated_csv_params(namespaces=["scores", namespace], **{flag: True}) + + assert resolved.namespaces == ["scores", namespace] + + def test_the_appended_tokens_are_the_published_spellings(self): + """These strings are the request vocabulary; the enum values are frozen for exactly this reason.""" + resolved = resolve_deprecated_csv_params(include_post_mapped_hgvs=True, include_custom_columns=True) + + assert resolved.namespaces == ["mavedb", "scores_custom"] + + def test_every_deprecated_param_can_be_combined(self): + resolved = resolve_deprecated_csv_params( + namespaces=["scores"], + drop_na_columns=True, + include_post_mapped_hgvs=True, + include_custom_columns=True, + ) + + assert resolved.namespaces == ["scores", CsvNamespace.REFERENCE_HGVS, CsvNamespace.SCORES_CUSTOM] + assert resolved.drop_unused_hgvs_columns is True + assert set(resolved.deprecations) == {"drop_na_columns", "include_post_mapped_hgvs", "include_custom_columns"} + + +class TestResponseHeaders: + """RFC 8594 headers are how a client discovers it is on a deprecated path.""" + + def test_a_single_deprecation_is_announced(self): + resolved = resolve_deprecated_csv_params(drop_na_columns=True) + + assert resolved.response_headers == { + "Deprecation": "true", + "Warning": '299 - "drop_na_columns is deprecated, use drop_unused_hgvs_columns"', + } + + def test_several_deprecations_are_joined_in_a_stable_order(self): + """Sorted, so the header does not churn with parameter order and can be asserted on.""" + resolved = resolve_deprecated_csv_params(include_custom_columns=True, include_post_mapped_hgvs=True) + + assert resolved.response_headers["Warning"] == ( + '299 - "include_custom_columns is deprecated, use namespaces=scores_custom;' + ' include_post_mapped_hgvs is deprecated, use namespaces=mavedb"' + ) From a895695c5d7644c9a4d2e4646c426bb3131d4040 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 16:49:53 -0700 Subject: [PATCH 7/9] fix(csv): default to empty dict in variant csv rather than asserting --- src/mavedb/lib/csv/score_set.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py index 9aafad32d..bbaf33c13 100644 --- a/src/mavedb/lib/csv/score_set.py +++ b/src/mavedb/lib/csv/score_set.py @@ -41,9 +41,10 @@ def get_score_set_variants_as_csv( viewer: Optional[ScoreCalibrationViewer] = None, ) -> str: """Get the variant data from a score set as a CSV string.""" - assert type(score_set.dataset_columns) is dict - - plan = plan_csv_columns(score_set.dataset_columns, namespaces) + # `dataset_columns` is NOT NULL with a `{}` default, so only a transient score set reaches this unset. + # Treating it as empty if it is unset yields the core columns alone rather than failing deeper in + # column planning. + plan = plan_csv_columns(score_set.dataset_columns or {}, namespaces) fetched = fetch_variant_csv_data( db, From 3e222a2e5505e44c1cf1748edd6dd58d77c49868 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 11 Aug 2026 09:27:05 -0700 Subject: [PATCH 8/9] fix(tests): add importorskips to relevant csv test modules --- tests/lib/csv/test_deprecated_params.py | 4 ++++ tests/lib/csv/test_entries.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/lib/csv/test_deprecated_params.py b/tests/lib/csv/test_deprecated_params.py index e5c032463..fa63fb30e 100644 --- a/tests/lib/csv/test_deprecated_params.py +++ b/tests/lib/csv/test_deprecated_params.py @@ -1,3 +1,5 @@ +# ruff: noqa: E402 + """The compatibility layer keeping pre-namespace CSV clients working. This is the only thing standing between a client that predates the namespace vocabulary and silently @@ -8,6 +10,8 @@ import pytest +pytest.importorskip("fastapi") + from mavedb.lib.csv.deprecated_params import resolve_deprecated_csv_params from mavedb.lib.csv.namespaces import CsvNamespace diff --git a/tests/lib/csv/test_entries.py b/tests/lib/csv/test_entries.py index 7a225c4ef..3708eb3ed 100644 --- a/tests/lib/csv/test_entries.py +++ b/tests/lib/csv/test_entries.py @@ -1,5 +1,9 @@ +# ruff: noqa: E402 + import pytest +pytest.importorskip("fastapi") + from mavedb.lib.csv.entries import clinvar_namespace_entries from mavedb.lib.csv.namespaces import CsvNamespaceGroup From 7843dce3f5f1366bf5f7a38a67d9455e57211f6b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 12 Aug 2026 09:53:39 -0700 Subject: [PATCH 9/9] fix(csv): report GA4GH id instead of bare digest for post-mapped VRS get_digest_from_post_mapped is renamed to get_id_from_post_mapped and now returns the VRS object's `id` field verbatim (e.g. `ga4gh:VA.{digest}`) instead of the bare `digest`. Only `id` is indexed by ix_mapped_variants_post_mapped_id and matched by GET /mapped-variants/vrs/{identifier}, so exporting the digest produced identifiers that could not be resolved back against MaveDB. - rename the CSV column mavedb.post_mapped_vrs_digest to mavedb.post_mapped_vrs_id and update README.md's column reference - never synthesize an id from a digest-only object, and leave nested VRS 1.x variation.id unread, matching the lookup endpoint's reach - add coverage for the digest/id divergence, the digest-only case, and the VRS 1.x nesting case --- src/mavedb/lib/csv/specs.py | 14 ++++--- src/mavedb/lib/variants.py | 20 ++++++--- src/mavedb/scripts/resources/README.md | 4 +- tests/lib/csv/test_columns.py | 2 +- tests/lib/csv/test_variant.py | 25 +++++++++++- tests/lib/test_variants.py | 56 ++++++++++++++++++++------ tests/routers/test_score_set.py | 4 +- 7 files changed, 96 insertions(+), 29 deletions(-) diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index b3b671598..157cbdf43 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -7,7 +7,7 @@ from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.lib.variants import get_hgvs_from_post_mapped, get_id_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant @@ -129,11 +129,15 @@ def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: return fallback if fallback is not None and is_hgvs_p(fallback) else None -def _post_mapped_vrs_digest(mapping: Optional[MappedVariant]) -> Optional[str]: - """The digest of the post-mapped VRS object, or None if there is no post-mapped object.""" +def _post_mapped_vrs_id(mapping: Optional[MappedVariant]) -> Optional[str]: + """The GA4GH identifier of the post-mapped VRS object, or None if there is no post-mapped object. + + The full ``ga4gh:VA.{digest}`` identifier rather than the bare digest, so a value copied out of an + export can be pasted straight into the VRS search, which validates against the GA4GH CURIE form. + """ if mapping is None or not mapping.post_mapped: return None - return get_digest_from_post_mapped(mapping.post_mapped) + return get_id_from_post_mapped(mapping.post_mapped) def _target_genes(variant: Variant) -> Optional[str]: @@ -179,7 +183,7 @@ def _optional(getter: Callable) -> Callable: "post_mapped_hgvs_p": _post_mapped_hgvs_p, "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), - "post_mapped_vrs_digest": _post_mapped_vrs_digest, + "post_mapped_vrs_id": _post_mapped_vrs_id, }, needs_mappings=True, ), diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index bef9725c0..487d101aa 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -51,21 +51,29 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: return variations_hgvs[0] -def get_digest_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: +def get_id_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: """ - Extract the digest value from a post-mapped VRS object. + Extract the GA4GH identifier from a post-mapped VRS object. + + Returns the stored ``id`` verbatim rather than the ``digest`` field. The two are not interchangeable: + ``id`` is the value indexed by ``ix_mapped_variants_post_mapped_id`` and matched by + ``GET /mapped-variants/vrs/{identifier}``, so reading it is what makes an exported identifier resolvable + against our own records.. + + Only the top-level ``id`` is consulted. Early VRS 1.3 objects nest the allele under a ``variation`` + property, and the lookup endpoint does not reach into that nesting either, so unwrapping it here + would emit identifiers no MaveDB query can resolve. Args: - post_mapped_vrs: A post-mapped VRS (Variation Representation Specification) object - that may contain a digest field. Can be None. + post_mapped_vrs: A post-mapped VRS (Variation Representation Specification) object. Can be None. Returns: - The digest string if present in the post_mapped_vrs object, otherwise None. + The GA4GH identifier (``ga4gh:VA.{digest}`` for an Allele), or None if the object carries none. """ if not post_mapped_vrs: return None - return post_mapped_vrs.get("digest") # type: ignore + return post_mapped_vrs.get("id") # type: ignore # TODO (https://github.com/VariantEffect/mavedb-api/issues/440) Temporarily, we are using these functions to distinguish diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 30d244a62..f59b4c845 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -76,7 +76,7 @@ identifies which data source a column belongs to and is separated from the colum | *(no prefix)* | Core identifiers — `accession`, `hgvs_nt`, `hgvs_pro`, `hgvs_splice` | | `scores.` | Score columns defined by the score set author (e.g. `scores.score`) | | `counts.` | Count columns defined by the score set author | -| `mavedb.` | Columns computed by the MaveDB mapping pipeline (post-mapped HGVS, VRS digest) | +| `mavedb.` | Columns computed by the MaveDB mapping pipeline (post-mapped HGVS, VRS identifier) | | `vep.` | Ensembl Variant Effect Predictor annotations | | `gnomad.` | gnomAD population frequency data | | `clingen.` | ClinGen Allele Registry linkage | @@ -128,7 +128,7 @@ Columns are grouped by a namespace prefix. The groups below always appear: | `mavedb.post_mapped_hgvs_c` | Post-mapped coding HGVS (c. notation) | | `mavedb.post_mapped_hgvs_p` | Post-mapped protein HGVS (p. notation) | | `mavedb.post_mapped_hgvs_at_assay_level` | Post-mapped HGVS at the assay reference level (transcript or protein) | -| `mavedb.post_mapped_vrs_digest` | GA4GH VRS digest identifier for the post-mapped allele | +| `mavedb.post_mapped_vrs_id` | GA4GH VRS identifier for the post-mapped allele (e.g. `ga4gh:VA.n9ax-9x6gOC0OEt73VMYqCBfqfxG1XUH`) | | `vep.vep_functional_consequence` | VEP functional consequence term (e.g. `missense_variant`) | | `gnomad.gnomad_af` | gnomAD v4.1 allele frequency | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index c00da889c..74dd588a9 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -357,7 +357,7 @@ def test_plan_csv_columns_reference_hgvs_namespace_populates_columns(): "post_mapped_hgvs_p", "post_mapped_hgvs_c", "post_mapped_hgvs_at_assay_level", - "post_mapped_vrs_digest", + "post_mapped_vrs_id", ] diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index fd205f1cc..9105e5041 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -23,8 +23,8 @@ from mavedb.models.acmg_classification import ACMGClassification from mavedb.models.clinical_control import ClinicalControl from mavedb.models.enums.acmg_criterion import ACMGCriterion -from mavedb.models.enums.user_role import UserRole from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.enums.user_role import UserRole from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration @@ -33,11 +33,14 @@ from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant from tests.helpers.constants import ( + TEST_GA4GH_DIGEST, + TEST_GA4GH_IDENTIFIER, TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, TEST_MINIMAL_MAPPED_VARIANT, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, ) # --------------------------------------------------------------------------- @@ -449,6 +452,7 @@ def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" + mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X mapped_variant.vep_functional_consequence = "missense_variant" mapped_variant.clingen_allele_id = "CA123456" mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) @@ -463,10 +467,29 @@ def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NP_000305.3:p.Lys34Glu" + assert rows[0]["mavedb.post_mapped_vrs_id"] == TEST_GA4GH_IDENTIFIER + assert rows[0]["mavedb.post_mapped_vrs_id"] != TEST_GA4GH_DIGEST assert rows[0]["vep.vep_functional_consequence"] == "missense_variant" assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) assert rows[0]["clingen.clingen_allele_id"] == "CA123456" + def test_post_mapped_vrs_id_is_never_synthesized_from_digest(self, session, setup_lib_db_with_mapped_variant): + """A post-mapped object carrying only a ``digest`` reports NA rather than a built-up CURIE. + + The two stored fields are known to disagree on some rows, and only ``id`` is indexed and matched + by the VRS lookup, so a digest-derived identifier would resolve to nothing. + """ + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.post_mapped = { + key: value for key, value in TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X.items() if key != "id" + } + session.add(mapped_variant) + session.commit() + + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert rows[0]["mavedb.post_mapped_vrs_id"] == "NA" + def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index ca9c2b0b3..f01650a05 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -1,13 +1,15 @@ import pytest from mavedb.lib.variants import ( - get_digest_from_post_mapped, get_hgvs_from_post_mapped, + get_id_from_post_mapped, hgvs_from_vrs_allele, is_hgvs_g, is_hgvs_p, ) from tests.helpers.constants import ( + TEST_GA4GH_DIGEST, + TEST_GA4GH_IDENTIFIER, TEST_HGVS_IDENTIFIER, TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X, TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, @@ -71,30 +73,60 @@ def test_get_hgvs_from_post_mapped_invalid_structure(): get_hgvs_from_post_mapped({"invalid_key": "InvalidType"}) -### Tests for get_digest_from_post_mapped function ### +### Tests for get_id_from_post_mapped function ### -def test_get_digest_from_post_mapped_with_digest(): - post_mapped_vrs = {"digest": "test_digest_value", "type": "Allele"} - result = get_digest_from_post_mapped(post_mapped_vrs) - assert result == "test_digest_value" +def test_get_id_from_post_mapped_with_id(): + result = get_id_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X) + assert result == TEST_GA4GH_IDENTIFIER -def test_get_digest_from_post_mapped_without_digest(): +def test_get_id_from_post_mapped_without_id(): post_mapped_vrs = {"type": "Allele", "other_field": "value"} - result = get_digest_from_post_mapped(post_mapped_vrs) + result = get_id_from_post_mapped(post_mapped_vrs) assert result is None -def test_get_digest_from_post_mapped_none_input(): - result = get_digest_from_post_mapped(None) +def test_get_id_from_post_mapped_prefers_id_over_digest(): + """The stored ``id`` is returned verbatim, never synthesized from the sibling ``digest``. + + The two fields are known to disagree on some rows, and only ``id`` is indexed and matched by the + VRS lookup endpoint, so a digest-derived identifier would resolve to nothing. + """ + post_mapped_vrs = {"type": "Allele", "id": TEST_GA4GH_IDENTIFIER, "digest": "a_different_digest_value_entirely"} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result == TEST_GA4GH_IDENTIFIER + + +def test_get_id_from_post_mapped_ignores_digest_when_id_absent(): + """A row with a digest but no ``id`` yields nothing, rather than a synthesized ``ga4gh:VA.`` CURIE.""" + post_mapped_vrs = {"type": "Allele", "digest": TEST_GA4GH_DIGEST} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result is None + + +def test_get_id_from_post_mapped_ignores_nested_vrs_1_x_id(): + """VRS 1.x ids nested under ``variation`` are not unwrapped, matching the lookup endpoint's reach.""" + post_mapped_vrs = {"type": "Allele", "variation": {"id": TEST_GA4GH_IDENTIFIER}} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result is None + + +def test_get_id_from_post_mapped_none_input(): + result = get_id_from_post_mapped(None) assert result is None -def test_get_digest_from_post_mapped_empty_dict(): - result = get_digest_from_post_mapped({}) +def test_get_id_from_post_mapped_empty_dict(): + result = get_id_from_post_mapped({}) assert result is None diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 7fb80c12c..f58aa940c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -3443,7 +3443,7 @@ def test_download_variants_data_file( "mavedb.post_mapped_hgvs_p", "mavedb.post_mapped_hgvs_c", "mavedb.post_mapped_hgvs_at_assay_level", - "mavedb.post_mapped_vrs_digest", + "mavedb.post_mapped_vrs_id", "scores.score", ] ) @@ -3756,7 +3756,7 @@ def test_download_scores_counts_and_post_mapped_variants_file( "mavedb.post_mapped_hgvs_g", "mavedb.post_mapped_hgvs_p", "mavedb.post_mapped_hgvs_at_assay_level", - "mavedb.post_mapped_vrs_digest", + "mavedb.post_mapped_vrs_id", "scores.score", "scores.s_0", "scores.s_1",