diff --git a/docs/architecture/platform.md b/docs/architecture/platform.md index a0b03b29..77c57b55 100644 --- a/docs/architecture/platform.md +++ b/docs/architecture/platform.md @@ -818,13 +818,49 @@ Results are normalized into `SearchHit`: - source/index generation - modality - start/end time -- score and score semantics +- `rank`, `score`, and `raw_distance` (see ranking values below) - displayable text/metadata - optional preview reference The application owns limits, score normalization, multimodal fusion and deterministic ordering. UI/API/MCP may request stricter limits but cannot loosen policy. +### Ranking values + +Every search response is self-describing about its numbers so callers never have to +guess which value they received or which direction ranks better. A `RetrievalScoring` +descriptor is attached to `SearchResult`, `FusedSearchResult`, and `QueryAnswer`, so a +fused search keeps the same meaning when it becomes a video-query answer, and the CLI, +HTTP, MCP, and stored job results all expose it. + +- `raw_distance` is the raw vector-store distance under `scoring.distance_metric` + (`l2`, `cosine`, or `ip`); smaller is closer. The metric is null when it was not + recorded, such as results saved before the descriptor existed, and a fused result + reports it only when every searched channel recorded the same metric. +- `score` on a hit is derived as `score = -raw_distance` (`score_transform`), so larger + ranks better. +- A `SearchHit.rank` is 1-based within one modality channel; ranks from different + channels are not comparable. +- A `FusedMoment.rank`/`score` is the combined position and reciprocal-rank-fusion score + across channels; `fusion.searched_modalities` lists the channels that ran and + `FusedMoment.modalities` lists the channels that contributed to that moment. +- All of these are `ordering_only` (`scoring.score_calibration` and + `fusion.score_calibration`): valid for sorting within a single response, never a + probability or confidence. Calibrated scores are deferred to the end-to-end ranking + evaluation in issue #76. + +Evidence items (`EvidenceBoardCandidate`, `EvidenceBoardTile`, and `EvidenceDeliveryItem`) +copy a `score` from their source, and `score_semantics` states which score it is: + +- `fused_moment` for search evidence: the moment's combined reciprocal-rank-fusion + score, described by the carried `fusion` provenance. +- `channel_hit` for video-query moment evidence: one channel's hit score, described by + the carried `scoring` descriptor. Compare it only with hits from the same channel in + the same response. + +`score_semantics` is null when the evidence has no score, such as actor evidence, or +when its source was not recorded. + Actor cluster and detection queries use the same pagination conventions. ## 16. Natural-language query layer diff --git a/src/vidxp/application_models.py b/src/vidxp/application_models.py index 257edcf8..64260507 100644 --- a/src/vidxp/application_models.py +++ b/src/vidxp/application_models.py @@ -892,6 +892,65 @@ class FusionProfile(StrEnum): reciprocal_rank = "rrf_v1" +class RankDirection(StrEnum): + """Whether smaller or larger values of a ranking quantity rank better.""" + + lower_is_better = "lower_is_better" + higher_is_better = "higher_is_better" + + +class ScoreCalibration(StrEnum): + """How much meaning a numeric score carries beyond ordering.""" + + ordering_only = "ordering_only" + + +class RetrievalScoring(ApplicationModel): + """Self-describing meaning of the ranking values in a search response. + + Callers see several numbers on a hit (a raw vector-store distance and a + derived score) and a rank. This descriptor states, in the payload itself, + which distance metric produced ``raw_distance``, which direction ranks + better, how ``score`` is derived from the distance, and that neither value + is calibrated. Search results and video-query answers carry the same + descriptor, and evidence copied from a channel hit carries it too, so the + meaning never drifts between surfaces. Calibrated, probability-like scores are + deferred to the end-to-end ranking evaluation tracked in issue #76. + """ + + distance_metric: Literal["l2", "cosine", "ip"] | None = Field( + default=None, + description=( + "Vector-store distance space that produced each raw_distance. Null " + "when the metric was not recorded, such as results saved before " + "this descriptor existed." + ), + ) + raw_distance_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="raw_distance sorts ascending; a smaller distance is closer.", + ) + score_transform: Literal["negated_distance"] = Field( + default="negated_distance", + description="Each hit score is derived as score = -raw_distance.", + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "raw_distance and score are ordering_only: valid for sorting hits " + "within one response, never a probability or confidence." + ), + ) + hit_rank_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="A hit's per-channel rank starts at 1; rank 1 is the closest.", + ) + + class EvidenceDeliveryMode(StrEnum): none = "none" keyframes = "keyframes" @@ -923,7 +982,21 @@ class EvidenceBoardCandidate(ApplicationModel): representative_timestamp: float = Field(ge=0) frame_index: int | None = Field(default=None, ge=0) frame_match: "EvidenceFrameMatch" - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Ordering-only score copied from the evidence source, never a " + "probability; larger ranks better. score_semantics states whether " + "it is a fused moment score or one channel's hit score." + ), + ) + score_semantics: "EvidenceScoreSemantics | None" = Field( + default=None, + description=( + "Source and meaning of score. Null when the evidence has no score " + "or its source was not recorded." + ), + ) display_text: str | None = Field(default=None, max_length=512) provenance: dict[str, JsonValue] = Field(default_factory=dict) @@ -933,6 +1006,12 @@ def _valid_interval(self) -> "EvidenceBoardCandidate": raise ValueError("Evidence board candidate end precedes its start.") return self + @model_validator(mode="after") + def _score_semantics_describe_a_score(self) -> "EvidenceBoardCandidate": + if self.score is None and self.score_semantics is not None: + raise ValueError("Evidence score semantics require a score.") + return self + class SearchCommand(ApplicationModel): query: SearchQuery = Field(description="Text to match against indexed moments.") @@ -978,14 +1057,32 @@ def _unique_modalities( class SearchHit(ApplicationModel): - rank: int = Field(gt=0) + rank: int = Field( + gt=0, + description=( + "1-based rank of this hit within its own modality channel " + "(rank 1 is closest). Ranks from different channels are not " + "comparable; the combined position is FusedMoment.rank." + ), + ) media_id: MediaId video_id: VideoId generation_id: IndexGenerationId start: float = Field(ge=0) end: float = Field(gt=0) - score: float - raw_distance: float + score: float = Field( + description=( + "Ordering-only channel score, score = -raw_distance, so larger " + "ranks better. Not a probability or confidence; see the response " + "scoring descriptor." + ), + ) + raw_distance: float = Field( + description=( + "Raw vector-store distance under the configured metric; smaller is " + "closer. See scoring.distance_metric for the metric." + ), + ) modality: str = Field(min_length=1) source_id: str = Field(min_length=1) metadata: dict[str, JsonValue] = Field(default_factory=dict) @@ -1033,6 +1130,10 @@ class SearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: str = Field(min_length=1) modality: str = Field(min_length=1) + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description="Meaning of the rank, score, and raw_distance on each hit.", + ) hits: tuple[SearchHit, ...] = () def to_dict(self) -> dict[str, Any]: @@ -1046,18 +1147,81 @@ class FusionProvenance(ApplicationModel): profile: Literal[FusionProfile.reciprocal_rank] = FusionProfile.reciprocal_rank rank_constant: int = Field(default=60, gt=0) overlap_rule: Literal["connected_intervals", "shared_overlap"] = "connected_intervals" - requested_modalities: tuple[Identifier, ...] = () - searched_modalities: tuple[Identifier, ...] = () + requested_modalities: tuple[Identifier, ...] = Field( + default=(), + description="Channels the caller asked to search.", + ) + searched_modalities: tuple[Identifier, ...] = Field( + default=(), + description=( + "Channels actually run for this response. A moment's contributing " + "channels are the subset in FusedMoment.modalities." + ), + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="FusedMoment.score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "The combined FusedMoment.score is ordering_only: valid for sorting " + "moments within one response, never a probability or confidence." + ), + ) + + +class FusedMomentScoreSemantics(ApplicationModel): + """An evidence score copied from a fused moment's combined score.""" + + source: Literal["fused_moment"] = "fused_moment" + fusion: FusionProvenance = Field( + description=( + "Fusion that produced the score: reciprocal-rank fusion over the " + "searched channels, ordering-only and higher-is-better." + ), + ) + + +class ChannelHitScoreSemantics(ApplicationModel): + """An evidence score copied from one channel's search hit.""" + + source: Literal["channel_hit"] = "channel_hit" + scoring: RetrievalScoring = Field( + description=( + "Retrieval descriptor for the hit score: score = -raw_distance, with " + "raw_distance measured under scoring.distance_metric. Ordering-only; " + "compare it only with other hits from the same channel in the same " + "response." + ), + ) + + +EvidenceScoreSemantics = Annotated[ + FusedMomentScoreSemantics | ChannelHitScoreSemantics, + Field(discriminator="source"), +] class FusedMoment(ApplicationModel): moment_id: Sha256 | None = None - rank: int = Field(gt=0) - score: float = Field(gt=0) + rank: int = Field( + gt=0, + description="1-based combined rank across all searched channels; rank 1 is best.", + ) + score: float = Field( + gt=0, + description=( + "Combined reciprocal-rank-fusion score; larger ranks better. " + "Ordering-only, not a probability; see fusion.score_calibration." + ), + ) media_id: MediaId start: float = Field(ge=0) end: float = Field(gt=0) - modalities: tuple[Identifier, ...] + modalities: tuple[Identifier, ...] = Field( + description="Channels that contributed a hit to this moment.", + ) hits: tuple[SearchHit, ...] = Field(min_length=1) @model_validator(mode="after") @@ -1076,6 +1240,14 @@ class FusedSearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: SearchQuery modalities: tuple[Identifier, ...] + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description=( + "Meaning of the per-hit rank, score, and raw_distance carried by " + "each moment's hits. The combined moment score is described by " + "fusion.score_calibration and fusion.score_direction." + ), + ) moments: tuple[FusedMoment, ...] = () fusion: FusionProvenance evidence_delivery: "EvidenceDeliveryResult | None" = None @@ -1260,7 +1432,21 @@ class EvidenceDeliveryItem(ApplicationModel): media_id: MediaId generation_id: IndexGenerationId modalities: tuple[Identifier, ...] = Field(min_length=1) - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Ordering-only score copied from the evidence source, never a " + "probability; larger ranks better. score_semantics states whether " + "it is a fused moment score or one channel's hit score." + ), + ) + score_semantics: EvidenceScoreSemantics | None = Field( + default=None, + description=( + "Source and meaning of score. Null when the evidence has no score " + "or its source was not recorded." + ), + ) provenance: dict[str, JsonValue] = Field(default_factory=dict) state: EvidenceDeliveryState range: EvidenceRangeResolution | None = None @@ -1268,6 +1454,12 @@ class EvidenceDeliveryItem(ApplicationModel): clip: EvidenceArtifact | None = None errors: tuple[ErrorDetail, ...] = () + @model_validator(mode="after") + def _score_semantics_describe_a_score(self) -> "EvidenceDeliveryItem": + if self.score is None and self.score_semantics is not None: + raise ValueError("Evidence score semantics require a score.") + return self + class EvidenceDeliveryResult(ApplicationModel): policy: EvidenceDeliveryPolicy @@ -1360,6 +1552,14 @@ class QueryAnswer(ApplicationModel): model: QueryModelIdentity | None = None claims: tuple[GroundedClaim, ...] = () evidence: tuple[Evidence, ...] = Field(default=(), max_length=200) + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description=( + "Meaning of the rank, score, and raw_distance on the hits in " + "evidence and moments, carried over from the fused search. The " + "combined moment score is described by fusion." + ), + ) moments: tuple[FusedMoment, ...] = () fusion: FusionProvenance evidence_delivery: EvidenceDeliveryResult | None = None diff --git a/src/vidxp/capabilities/search.py b/src/vidxp/capabilities/search.py index 9c57ddc3..8505d710 100644 --- a/src/vidxp/capabilities/search.py +++ b/src/vidxp/capabilities/search.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Mapping +from vidxp.application_models import RetrievalScoring from vidxp.capabilities.schemas import SearchHit, SearchResult from vidxp.core.contracts import ( IndexConfig, @@ -143,6 +144,7 @@ def search_embeddings( query_id=query_id or stable_query_id(query, modality, config), query=query, modality=modality, + scoring=RetrievalScoring(distance_metric=config.vector_distance), hits=_to_hits(modality, rows, required_metadata), ) diff --git a/src/vidxp/cli_support.py b/src/vidxp/cli_support.py index 934c3b5d..844445cf 100644 --- a/src/vidxp/cli_support.py +++ b/src/vidxp/cli_support.py @@ -147,11 +147,18 @@ def emit_search( if not result.moments: typer.echo("No matching moments found.") return - table = Table(title="Fused search results") + table = Table( + title="Fused search results", + caption=( + "Rank 1 = best match. Score = reciprocal-rank fusion " + "(higher ranks better); ordering-only, not a probability. " + f"Distance metric: {result.scoring.distance_metric or 'not recorded'}." + ), + ) table.add_column("Rank", justify="right") table.add_column("Start", justify="right") table.add_column("End", justify="right") - table.add_column("Score", justify="right") + table.add_column("Score (RRF)", justify="right") table.add_column("Video") table.add_column("Modalities") for moment in result.moments: diff --git a/src/vidxp/evidence_delivery.py b/src/vidxp/evidence_delivery.py index ec01cbe3..c25dcff2 100644 --- a/src/vidxp/evidence_delivery.py +++ b/src/vidxp/evidence_delivery.py @@ -5,6 +5,7 @@ from vidxp.application_models import ( ApplicationError, + ChannelHitScoreSemantics, CreateSnippetCommand, ErrorCategory, ErrorDetail, @@ -19,6 +20,7 @@ EvidenceFrameMatch, EvidenceKeyframe, EvidenceRangeResolution, + FusedMomentScoreSemantics, FusedSearchResult, Job, JobKind, @@ -163,6 +165,7 @@ def _search_candidates( result: FusedSearchResult, ) -> tuple[EvidenceBoardCandidate, ...]: candidates: list[EvidenceBoardCandidate] = [] + score_semantics = FusedMomentScoreSemantics(fusion=result.fusion) for moment in result.moments: if moment.moment_id is None: continue @@ -193,6 +196,7 @@ def _search_candidates( else EvidenceFrameMatch.representative ), score=moment.score, + score_semantics=score_semantics, display_text=EvidenceDeliveryService._display_text( selected.metadata ), @@ -217,6 +221,7 @@ def _query_candidates( answer: QueryAnswer, ) -> tuple[EvidenceBoardCandidate, ...]: candidates: list[EvidenceBoardCandidate] = [] + hit_score_semantics = ChannelHitScoreSemantics(scoring=answer.scoring) for rank, evidence in enumerate(answer.evidence, start=1): if isinstance(evidence, MomentEvidence): raw_index = evidence.hit.metadata.get("frame_index") @@ -233,6 +238,7 @@ def _query_candidates( else (evidence.start + evidence.end) / 2 ) score = evidence.hit.score + score_semantics = hit_score_semantics provenance = { "source_id": evidence.source_id, "kind": evidence.kind, @@ -241,6 +247,7 @@ def _query_candidates( frame_index = None representative = (evidence.start + evidence.end) / 2 score = None + score_semantics = None provenance = { "cluster_id": evidence.cluster_id, "detection_count": evidence.detection_count, @@ -264,6 +271,7 @@ def _query_candidates( else EvidenceFrameMatch.representative ), score=score, + score_semantics=score_semantics, display_text=evidence.display_text, provenance=provenance, ) @@ -439,6 +447,7 @@ def _deliver( generation_id=candidate.generation_id, modalities=candidate.modalities, score=candidate.score, + score_semantics=candidate.score_semantics, provenance=candidate.provenance, state=EvidenceDeliveryState.failed, errors=( @@ -525,6 +534,7 @@ def _deliver( generation_id=candidate.generation_id, modalities=candidate.modalities, score=candidate.score, + score_semantics=candidate.score_semantics, provenance=candidate.provenance, state=( EvidenceDeliveryState.partial diff --git a/src/vidxp/query_service.py b/src/vidxp/query_service.py index c8d0aaf0..11ea34ac 100644 --- a/src/vidxp/query_service.py +++ b/src/vidxp/query_service.py @@ -204,6 +204,7 @@ def answer( "plan": plan, "model": self.model.identity if self.model is not None else None, "evidence": evidence, + "scoring": fused.scoring, "moments": fused.moments, "fusion": fused.fusion, } diff --git a/src/vidxp/search_fusion.py b/src/vidxp/search_fusion.py index 6af5597d..a4c21749 100644 --- a/src/vidxp/search_fusion.py +++ b/src/vidxp/search_fusion.py @@ -7,6 +7,7 @@ FusedMoment, FusedSearchResult, FusionProvenance, + RetrievalScoring, SearchHit, SearchResult, ) @@ -100,6 +101,20 @@ def _score(hits: list[SearchHit]) -> float: return sum(1.0 / (RRF_RANK_CONSTANT + rank) for rank in best_ranks.values()) +def _shared_scoring(results: tuple[SearchResult, ...]) -> RetrievalScoring: + """Describe fused hits without turning an unrecorded metric into a known one. + + Channels searched together share one index and normally record the same + metric. The metric is reported only when every channel recorded it + identically; otherwise it stays unknown. + """ + + metrics = {result.scoring.distance_metric for result in results} + return RetrievalScoring( + distance_metric=next(iter(metrics)) if len(metrics) == 1 else None + ) + + def _moment_id( *, snapshot_id: str | None, @@ -214,6 +229,7 @@ def fuse_search_results( ), query=query, modalities=searched_modalities, + scoring=_shared_scoring(ordered_results), moments=moments, fusion=FusionProvenance( overlap_rule="shared_overlap", diff --git a/tests/test_evidence_delivery.py b/tests/test_evidence_delivery.py index 4c43f112..d672ca29 100644 --- a/tests/test_evidence_delivery.py +++ b/tests/test_evidence_delivery.py @@ -8,9 +8,15 @@ from types import SimpleNamespace from unittest.mock import Mock +from pydantic import ValidationError + from vidxp.application_models import ( + ActorEvidence, ApplicationError, Artifact, + EvidenceBoardCandidate, + EvidenceBoardTile, + EvidenceDeliveryItem, EvidenceDeliveryMode, EvidenceDeliveryPolicy, EvidenceDeliveryState, @@ -22,6 +28,7 @@ QueryAnswer, QueryAnswerMode, QueryPlan, + RetrievalScoring, SearchMomentsPlanStep, SearchHit, SearchResult, @@ -398,6 +405,145 @@ def test_query_delivery_preserves_existing_evidence_identity(self): delivered.evidence[0].evidence_id, ) + def test_search_evidence_score_is_described_as_the_fused_moment_score(self): + service, _artifacts = self.service() + result = service.deliver_search( + fused(), + EvidenceDeliveryPolicy( + mode=EvidenceDeliveryMode.keyframes, + max_items=1, + ), + execution=ExecutionContext(job_id=JOB_ID), + ) + + moment = result.moments[0] + candidate = EvidenceDeliveryService.candidates(result)[0] + item = result.evidence_delivery.items[0] + for evidence in (candidate, item): + self.assertEqual(evidence.score, moment.score) + self.assertEqual(evidence.score_semantics.source, "fused_moment") + self.assertEqual(evidence.score_semantics.fusion, result.fusion) + + def test_query_evidence_score_is_described_as_the_channel_hit_score(self): + service, _artifacts = self.service() + hit = SearchHit.model_validate( + {**scene_hit().model_dump(), "score": -1.0, "raw_distance": 1.0} + ) + atomic = SearchResult( + query_id="scene:known", + query="green frame", + modality="scene", + scoring=RetrievalScoring(distance_metric="cosine"), + hits=(hit,), + ) + fused_result = fuse_search_results( + query="green frame", + requested_modalities=("scene",), + results=(atomic,), + snapshot_id=SNAPSHOT_ID, + ) + answer = QueryAnswer( + question="Which frame is green?", + mode=QueryAnswerMode.evidence_only, + plan=QueryPlan( + steps=(SearchMomentsPlanStep(modality="scene", query="green frame"),) + ), + evidence=( + MomentEvidence( + evidence_id="d" * 64, + snapshot_id=SNAPSHOT_ID, + media_id=MEDIA_ID, + generation_id=GENERATION_ID, + modality="scene", + source_id=hit.source_id, + start=hit.start, + end=hit.end, + hit=hit, + ), + ActorEvidence( + evidence_id="e" * 64, + snapshot_id=SNAPSHOT_ID, + media_id=MEDIA_ID, + generation_id=GENERATION_ID, + cluster_id="cluster-1", + start=1.0, + end=2.0, + detection_count=3, + display_text="Actor cluster cluster-1 appears 3 times.", + ), + ), + scoring=fused_result.scoring, + moments=fused_result.moments, + fusion=fused_result.fusion, + fallback_reason="query_model_not_configured", + ) + + delivered = service.deliver_query( + answer, + EvidenceDeliveryPolicy( + mode=EvidenceDeliveryMode.keyframes, + max_items=2, + ), + execution=ExecutionContext(job_id=JOB_ID), + ) + + # The fused moment score differs from the channel score copied into + # video-query evidence, so the payload must say which one it carries. + self.assertAlmostEqual(fused_result.moments[0].score, 1 / 61) + moment_candidate, actor_candidate = EvidenceDeliveryService.candidates( + delivered + ) + moment_item, actor_item = delivered.evidence_delivery.items + for evidence in (moment_candidate, moment_item): + self.assertEqual(evidence.score, -1.0) + self.assertEqual(evidence.score_semantics.source, "channel_hit") + scoring = evidence.score_semantics.scoring + self.assertEqual(scoring.distance_metric, "cosine") + self.assertEqual(scoring.score_transform, "negated_distance") + for evidence in (actor_candidate, actor_item): + self.assertIsNone(evidence.score) + self.assertIsNone(evidence.score_semantics) + + def test_evidence_score_semantics_require_a_score(self): + candidate = EvidenceDeliveryService.candidates(fused())[0] + + with self.assertRaises(ValidationError): + EvidenceBoardCandidate.model_validate( + {**candidate.model_dump(), "score": None} + ) + with self.assertRaises(ValidationError): + EvidenceDeliveryItem( + evidence_id=candidate.evidence_id, + rank=1, + media_id=MEDIA_ID, + generation_id=GENERATION_ID, + modalities=("scene",), + score=None, + score_semantics=candidate.score_semantics, + state=EvidenceDeliveryState.ready, + ) + + def test_board_tiles_keep_score_semantics_and_legacy_evidence_loads(self): + candidate = EvidenceDeliveryService.candidates(fused())[0] + + # Board tiles are built from candidates the same way the board job does. + tile = EvidenceBoardTile( + **candidate.model_dump(), + tile_id="f" * 64, + page_number=1, + position=1, + state=EvidenceDeliveryState.ready, + ) + self.assertEqual(tile.score_semantics, candidate.score_semantics) + + # A candidate stored before score semantics existed still loads, and its + # source stays unrecorded instead of being guessed. + legacy = candidate.model_dump(mode="json") + legacy.pop("score_semantics") + restored = EvidenceBoardCandidate.model_validate(legacy) + self.assertEqual(restored.score, candidate.score) + self.assertIsNone(restored.score_semantics) + @unittest.skipUnless(shutil.which("ffmpeg"), "ffmpeg is required") class ExactFrameRendererTests(unittest.TestCase): diff --git a/tests/test_query_service.py b/tests/test_query_service.py index e1fa3cd9..b532d393 100644 --- a/tests/test_query_service.py +++ b/tests/test_query_service.py @@ -6,14 +6,18 @@ FusedSearchResult, FusionProvenance, IndexSnapshotReference, + QueryAnswer, QueryAnswerMode, QueryModelIdentity, QueryPlan, QueryVideoCommand, + RetrievalScoring, SearchHit, SearchMomentsPlanStep, SearchResult, ) +from vidxp.capabilities.search import search_embeddings +from vidxp.core.contracts import IndexConfig from vidxp.ports import QueryProviderError from vidxp.query_service import GroundedQueryService from vidxp.search_fusion import fuse_search_results @@ -372,6 +376,94 @@ def test_evidence_is_bounded_to_retained_fused_moments(self): all(item.source_id in retained_sources for item in evidence) ) + def test_query_answer_keeps_cosine_retrieval_scoring(self): + class CosineStore: + def query(self, modality, embedding, **options): + return [ + { + "source_id": "speech:1", + "raw_distance": 0.25, + "metadata": { + "video_id": MEDIA_ID, + "generation_id": GENERATION_ID, + "start": 1.0, + "end": 2.0, + "text": "the taxi arrived", + }, + } + ] + + config = IndexConfig( + dataset="sample", + split="test", + run_id="run-1", + enabled_modalities=("speech",), + vector_distance="cosine", + ) + atomic = search_embeddings( + "taxi", + "speech", + [0.5, 0.25], + config=config, + required_metadata=frozenset({"text"}), + storage=CosineStore(), + ) + fused_result = fused("speech", atomic) + service = GroundedQueryService() + evidence = service.evidence( + snapshot=self.snapshot, + fused=fused_result, + actors=(), + ) + + answer = service.answer( + self.command, + plan=QueryPlan( + steps=(SearchMomentsPlanStep(modality="speech", query="taxi"),) + ), + planning_fallback="query_model_not_configured", + evidence=evidence, + fused=fused_result, + ) + + self.assertEqual(answer.scoring, fused_result.scoring) + self.assertEqual(answer.scoring.distance_metric, "cosine") + self.assertEqual(answer.scoring.score_transform, "negated_distance") + hit = answer.evidence[0].hit + self.assertEqual(hit.score, -hit.raw_distance) + # Stored query job results keep the descriptor too. + restored = QueryAnswer.model_validate_json(answer.model_dump_json()) + self.assertEqual(restored.scoring.distance_metric, "cosine") + + def test_legacy_query_answer_without_scoring_keeps_metric_unknown(self): + atomic = result(text="the taxi arrived", modality="speech").model_copy( + update={"scoring": RetrievalScoring(distance_metric="cosine")} + ) + fused_result = fused("speech", atomic) + service = GroundedQueryService() + evidence = service.evidence( + snapshot=self.snapshot, + fused=fused_result, + actors=(), + ) + answer = service.answer( + self.command, + plan=QueryPlan( + steps=(SearchMomentsPlanStep(modality="speech", query="taxi"),) + ), + planning_fallback="query_model_not_configured", + evidence=evidence, + fused=fused_result, + ) + self.assertEqual(answer.scoring.distance_metric, "cosine") + + # Simulate a query job result stored before the descriptor existed. + payload = answer.model_dump(mode="json") + payload.pop("scoring") + restored = QueryAnswer.model_validate(payload) + + self.assertIsNone(restored.scoring.distance_metric) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_search.py b/tests/test_search.py index accb7d84..2eedb98f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -112,6 +112,52 @@ def test_score_is_strictly_monotonic_and_not_a_probability(self): self.assertGreater(distance_to_score(0.1), distance_to_score(0.2)) self.assertEqual(distance_to_score(2.5), -2.5) + def test_result_describes_its_metric_and_ordering_only_scores(self): + config = IndexConfig( + dataset="sample", + split="test", + run_id="run-1", + enabled_modalities=("speech",), + vector_distance="cosine", + ) + storage = FakeStorage([dialogue_row("run:video-1:dialogue:a", 0.1)]) + with patch( + "vidxp.capabilities.speech.operations.speech_embedding", + return_value=[0.5, 0.25], + ): + result = search_speech( + "fresh bread", + config=config, + runtime=self.runtime, + top_k=1, + storage=storage, + ) + + scoring = result.scoring + self.assertEqual(scoring.distance_metric, "cosine") + self.assertEqual(scoring.raw_distance_direction, "lower_is_better") + self.assertEqual(scoring.score_transform, "negated_distance") + self.assertEqual(scoring.score_direction, "higher_is_better") + self.assertEqual(scoring.score_calibration, "ordering_only") + self.assertEqual(scoring.hit_rank_direction, "lower_is_better") + + def test_default_distance_metric_is_recorded_explicitly(self): + storage = FakeStorage([dialogue_row("run:video-1:dialogue:a", 0.1)]) + with patch( + "vidxp.capabilities.speech.operations.speech_embedding", + return_value=[0.5, 0.25], + ): + result = search_speech( + "fresh bread", + config=self.config, + runtime=self.runtime, + top_k=1, + storage=storage, + ) + + self.assertEqual(self.config.vector_distance, "l2") + self.assertEqual(result.scoring.distance_metric, "l2") + def test_dialogue_query_uses_model_owned_query_prompt(self): encoder = Mock() encoder.encode_query.return_value = np.asarray([[0.5, 0.25]]) diff --git a/tests/test_search_fusion.py b/tests/test_search_fusion.py index b7b193b6..d6495d6f 100644 --- a/tests/test_search_fusion.py +++ b/tests/test_search_fusion.py @@ -1,11 +1,18 @@ import unittest -from vidxp.application_models import FusedSearchResult, SearchHit, SearchResult +from vidxp.application_models import ( + FusedSearchResult, + RetrievalScoring, + SearchHit, + SearchResult, +) from vidxp.search_fusion import ( RRF_RANK_CONSTANT, fuse_search_results, resolve_candidate_limit, ) + + MEDIA_ID = "123456781234423481234567890abcde" GENERATION_ID = "223456781234423481234567890abcde" @@ -260,6 +267,97 @@ def test_nearby_duplicate_hits_combine_into_one_moment(self): self.assertEqual(result.moments[0].start, 2.2) self.assertEqual(result.moments[0].end, 2.8) + def test_fused_result_marks_every_score_ordering_only(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_direction, "higher_is_better") + + def test_fused_result_inherits_the_channel_distance_metric(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + scoring=RetrievalScoring(distance_metric="cosine"), + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.distance_metric, "cosine") + + def test_legacy_stored_result_without_scoring_still_loads(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + # Simulate a job result stored before the scoring descriptor existed. + payload = result.model_dump(mode="json") + payload.pop("scoring") + payload["fusion"].pop("score_direction") + payload["fusion"].pop("score_calibration") + + restored = FusedSearchResult.model_validate(payload) + + # The metric was never recorded, so it must stay unknown rather than + # silently becoming a definite metric. + self.assertIsNone(restored.scoring.distance_metric) + self.assertEqual(restored.scoring.score_calibration, "ordering_only") + self.assertEqual(restored.fusion.score_calibration, "ordering_only") + + def test_fused_metric_is_unknown_unless_every_channel_agrees(self): + def channel(modality, metric): + return SearchResult( + query_id=f"{modality}:q", + query="taxi", + modality=modality, + scoring=RetrievalScoring(distance_metric=metric), + hits=(hit(modality, 1, 1, 2, f"{modality}:1"),), + ) + + def fused_metric(*results): + return fuse_search_results( + query="taxi", + requested_modalities=tuple(result.modality for result in results), + results=results, + ).scoring.distance_metric + + self.assertEqual( + fused_metric(channel("scene", "cosine"), channel("speech", "cosine")), + "cosine", + ) + self.assertIsNone( + fused_metric(channel("scene", "cosine"), channel("speech", None)) + ) + self.assertIsNone( + fused_metric(channel("scene", "cosine"), channel("speech", "ip")) + ) + self.assertIsNone(fused_metric()) + if __name__ == "__main__": unittest.main()