From f3fba922ba584cf116148a0982a8c9a6c286790e Mon Sep 17 00:00:00 2001 From: AhmedIrfan7 Date: Sat, 5 Sep 2026 00:35:56 +0500 Subject: [PATCH 1/2] feat(search): make retrieval and fusion scores self-describing Search responses returned several ranking numbers (a raw vector-store distance, a derived per-channel score, a per-channel rank, and a combined fusion score) with no statement of what each meant or which direction ranked better, and the derived scores could be mistaken for calibrated confidences. Add a RetrievalScoring descriptor, carried on both SearchResult and FusedSearchResult, that states the distance metric, the low/high ranking direction of raw_distance and score, the distance->score transform (negated_distance), and that both values are ordering_only rather than a probability. Extend FusionProvenance to declare the combined moment score as ordering_only and higher-is-better, and document requested vs searched channels. Add field descriptions to every ranking value on SearchHit, FusedMoment, and the evidence artifacts so the meaning is identical across the CLI, HTTP, MCP, stored job results, and evidence delivery. The CLI search table now labels the score column and captions the metric and ordering-only meaning. The change is additive: every new field is defaulted, so previously stored job results and existing FusedSearchResult payloads still validate. Scores remain uncalibrated; calibrated scoring is deferred to the end-to-end ranking evaluation in #76. Closes #90 --- docs/architecture/platform.md | 24 ++++- src/vidxp/application_models.py | 148 ++++++++++++++++++++++++++++--- src/vidxp/capabilities/search.py | 2 + src/vidxp/cli_support.py | 11 ++- src/vidxp/search_fusion.py | 4 + tests/test_search.py | 29 ++++++ tests/test_search_fusion.py | 67 +++++++++++++- 7 files changed, 271 insertions(+), 14 deletions(-) diff --git a/docs/architecture/platform.md b/docs/architecture/platform.md index a0b03b29..7b73c565 100644 --- a/docs/architecture/platform.md +++ b/docs/architecture/platform.md @@ -818,13 +818,35 @@ Results are normalized into `SearchHit`: - source/index generation - modality - start/end time -- score and score semantics +- `rank`, `score`, and `raw_distance` (see ranking values below) - displayable text/metadata - optional preview reference The application owns limits, score normalization, multimodal fusion and deterministic ordering. UI/API/MCP may request stricter limits but cannot loosen policy. +### Ranking values + +Every search response is self-describing about its numbers so callers never have to +guess which value they received or which direction ranks better. A `RetrievalScoring` +descriptor is attached to both `SearchResult` and `FusedSearchResult`, and the same +descriptor is returned identically across the CLI, HTTP, MCP, stored job results, and +evidence artifacts. + +- `raw_distance` is the raw vector-store distance under `scoring.distance_metric` + (`l2`, `cosine`, or `ip`); smaller is closer. +- `score` on a hit is derived as `score = -raw_distance` (`score_transform`), so larger + ranks better. +- A `SearchHit.rank` is 1-based within one modality channel; ranks from different + channels are not comparable. +- A `FusedMoment.rank`/`score` is the combined position and reciprocal-rank-fusion score + across channels; `fusion.searched_modalities` lists the channels that ran and + `FusedMoment.modalities` lists the channels that contributed to that moment. +- All of these are `ordering_only` (`scoring.score_calibration` and + `fusion.score_calibration`): valid for sorting within a single response, never a + probability or confidence. Calibrated scores are deferred to the end-to-end ranking + evaluation in issue #76. + Actor cluster and detection queries use the same pagination conventions. ## 16. Natural-language query layer diff --git a/src/vidxp/application_models.py b/src/vidxp/application_models.py index 395bde42..cffe6a53 100644 --- a/src/vidxp/application_models.py +++ b/src/vidxp/application_models.py @@ -809,6 +809,61 @@ class FusionProfile(StrEnum): reciprocal_rank = "rrf_v1" +class RankDirection(StrEnum): + """Whether smaller or larger values of a ranking quantity rank better.""" + + lower_is_better = "lower_is_better" + higher_is_better = "higher_is_better" + + +class ScoreCalibration(StrEnum): + """How much meaning a numeric score carries beyond ordering.""" + + ordering_only = "ordering_only" + + +class RetrievalScoring(ApplicationModel): + """Self-describing meaning of the ranking values in a search response. + + Callers see several numbers on a hit (a raw vector-store distance and a + derived score) and a rank. This descriptor states, in the payload itself, + which distance metric produced ``raw_distance``, which direction ranks + better, how ``score`` is derived from the distance, and that neither value + is calibrated. It is returned identically across the CLI, HTTP, MCP, + stored job results, and evidence artifacts so the meaning never drifts + between surfaces. Calibrated, probability-like scores are deferred to the + end-to-end ranking evaluation tracked in issue #76. + """ + + distance_metric: Literal["l2", "cosine", "ip"] = Field( + default="l2", + description="Vector-store distance space that produced each raw_distance.", + ) + raw_distance_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="raw_distance sorts ascending; a smaller distance is closer.", + ) + score_transform: Literal["negated_distance"] = Field( + default="negated_distance", + description="Each hit score is derived as score = -raw_distance.", + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "raw_distance and score are ordering_only: valid for sorting hits " + "within one response, never a probability or confidence." + ), + ) + hit_rank_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="A hit's per-channel rank starts at 1; rank 1 is the closest.", + ) + + class EvidenceDeliveryMode(StrEnum): none = "none" keyframes = "keyframes" @@ -840,7 +895,13 @@ class EvidenceBoardCandidate(ApplicationModel): representative_timestamp: float = Field(ge=0) frame_index: int | None = Field(default=None, ge=0) frame_match: "EvidenceFrameMatch" - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Combined ordering-only fusion score copied from the source moment; " + "larger ranks better, not a probability." + ), + ) display_text: str | None = Field(default=None, max_length=512) provenance: dict[str, JsonValue] = Field(default_factory=dict) @@ -895,14 +956,32 @@ def _unique_modalities( class SearchHit(ApplicationModel): - rank: int = Field(gt=0) + rank: int = Field( + gt=0, + description=( + "1-based rank of this hit within its own modality channel " + "(rank 1 is closest). Ranks from different channels are not " + "comparable; the combined position is FusedMoment.rank." + ), + ) media_id: MediaId video_id: VideoId generation_id: IndexGenerationId start: float = Field(ge=0) end: float = Field(gt=0) - score: float - raw_distance: float + score: float = Field( + description=( + "Ordering-only channel score, score = -raw_distance, so larger " + "ranks better. Not a probability or confidence; see the response " + "scoring descriptor." + ), + ) + raw_distance: float = Field( + description=( + "Raw vector-store distance under the configured metric; smaller is " + "closer. See scoring.distance_metric for the metric." + ), + ) modality: str = Field(min_length=1) source_id: str = Field(min_length=1) metadata: dict[str, JsonValue] = Field(default_factory=dict) @@ -950,6 +1029,10 @@ class SearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: str = Field(min_length=1) modality: str = Field(min_length=1) + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description="Meaning of the rank, score, and raw_distance on each hit.", + ) hits: tuple[SearchHit, ...] = () def to_dict(self) -> dict[str, Any]: @@ -963,18 +1046,49 @@ class FusionProvenance(ApplicationModel): profile: Literal[FusionProfile.reciprocal_rank] = FusionProfile.reciprocal_rank rank_constant: int = Field(default=60, gt=0) overlap_rule: Literal["connected_intervals"] = "connected_intervals" - requested_modalities: tuple[Identifier, ...] = () - searched_modalities: tuple[Identifier, ...] = () + requested_modalities: tuple[Identifier, ...] = Field( + default=(), + description="Channels the caller asked to search.", + ) + searched_modalities: tuple[Identifier, ...] = Field( + default=(), + description=( + "Channels actually run for this response. A moment's contributing " + "channels are the subset in FusedMoment.modalities." + ), + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="FusedMoment.score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "The combined FusedMoment.score is ordering_only: valid for sorting " + "moments within one response, never a probability or confidence." + ), + ) class FusedMoment(ApplicationModel): moment_id: Sha256 | None = None - rank: int = Field(gt=0) - score: float = Field(gt=0) + rank: int = Field( + gt=0, + description="1-based combined rank across all searched channels; rank 1 is best.", + ) + score: float = Field( + gt=0, + description=( + "Combined reciprocal-rank-fusion score; larger ranks better. " + "Ordering-only, not a probability; see fusion.score_calibration." + ), + ) media_id: MediaId start: float = Field(ge=0) end: float = Field(gt=0) - modalities: tuple[Identifier, ...] + modalities: tuple[Identifier, ...] = Field( + description="Channels that contributed a hit to this moment.", + ) hits: tuple[SearchHit, ...] = Field(min_length=1) @model_validator(mode="after") @@ -993,6 +1107,14 @@ class FusedSearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: SearchQuery modalities: tuple[Identifier, ...] + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description=( + "Meaning of the per-hit rank, score, and raw_distance carried by " + "each moment's hits. The combined moment score is described by " + "fusion.score_calibration and fusion.score_direction." + ), + ) moments: tuple[FusedMoment, ...] = () fusion: FusionProvenance evidence_delivery: "EvidenceDeliveryResult | None" = None @@ -1177,7 +1299,13 @@ class EvidenceDeliveryItem(ApplicationModel): media_id: MediaId generation_id: IndexGenerationId modalities: tuple[Identifier, ...] = Field(min_length=1) - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Combined ordering-only fusion score copied from the source moment; " + "larger ranks better, not a probability." + ), + ) provenance: dict[str, JsonValue] = Field(default_factory=dict) state: EvidenceDeliveryState range: EvidenceRangeResolution | None = None diff --git a/src/vidxp/capabilities/search.py b/src/vidxp/capabilities/search.py index 9c57ddc3..8505d710 100644 --- a/src/vidxp/capabilities/search.py +++ b/src/vidxp/capabilities/search.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Mapping +from vidxp.application_models import RetrievalScoring from vidxp.capabilities.schemas import SearchHit, SearchResult from vidxp.core.contracts import ( IndexConfig, @@ -143,6 +144,7 @@ def search_embeddings( query_id=query_id or stable_query_id(query, modality, config), query=query, modality=modality, + scoring=RetrievalScoring(distance_metric=config.vector_distance), hits=_to_hits(modality, rows, required_metadata), ) diff --git a/src/vidxp/cli_support.py b/src/vidxp/cli_support.py index 934c3b5d..eadf8f2c 100644 --- a/src/vidxp/cli_support.py +++ b/src/vidxp/cli_support.py @@ -147,11 +147,18 @@ def emit_search( if not result.moments: typer.echo("No matching moments found.") return - table = Table(title="Fused search results") + table = Table( + title="Fused search results", + caption=( + "Rank 1 = best match. Score = reciprocal-rank fusion " + "(higher ranks better); ordering-only, not a probability. " + f"Distance metric: {result.scoring.distance_metric}." + ), + ) table.add_column("Rank", justify="right") table.add_column("Start", justify="right") table.add_column("End", justify="right") - table.add_column("Score", justify="right") + table.add_column("Score (RRF)", justify="right") table.add_column("Video") table.add_column("Modalities") for moment in result.moments: diff --git a/src/vidxp/search_fusion.py b/src/vidxp/search_fusion.py index b7e41849..058f74b5 100644 --- a/src/vidxp/search_fusion.py +++ b/src/vidxp/search_fusion.py @@ -7,6 +7,7 @@ FusedMoment, FusedSearchResult, FusionProvenance, + RetrievalScoring, SearchHit, SearchResult, ) @@ -190,6 +191,9 @@ def fuse_search_results( ), query=query, modalities=searched_modalities, + scoring=( + ordered_results[0].scoring if ordered_results else RetrievalScoring() + ), moments=moments, fusion=FusionProvenance( requested_modalities=requested_modalities, diff --git a/tests/test_search.py b/tests/test_search.py index accb7d84..6b43595f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -112,6 +112,35 @@ def test_score_is_strictly_monotonic_and_not_a_probability(self): self.assertGreater(distance_to_score(0.1), distance_to_score(0.2)) self.assertEqual(distance_to_score(2.5), -2.5) + def test_result_describes_its_metric_and_ordering_only_scores(self): + config = IndexConfig( + dataset="sample", + split="test", + run_id="run-1", + enabled_modalities=("speech",), + vector_distance="cosine", + ) + storage = FakeStorage([dialogue_row("run:video-1:dialogue:a", 0.1)]) + with patch( + "vidxp.capabilities.speech.operations.speech_embedding", + return_value=[0.5, 0.25], + ): + result = search_speech( + "fresh bread", + config=config, + runtime=self.runtime, + top_k=1, + storage=storage, + ) + + scoring = result.scoring + self.assertEqual(scoring.distance_metric, "cosine") + self.assertEqual(scoring.raw_distance_direction, "lower_is_better") + self.assertEqual(scoring.score_transform, "negated_distance") + self.assertEqual(scoring.score_direction, "higher_is_better") + self.assertEqual(scoring.score_calibration, "ordering_only") + self.assertEqual(scoring.hit_rank_direction, "lower_is_better") + def test_dialogue_query_uses_model_owned_query_prompt(self): encoder = Mock() encoder.encode_query.return_value = np.asarray([[0.5, 0.25]]) diff --git a/tests/test_search_fusion.py b/tests/test_search_fusion.py index 622e0210..2f603a6e 100644 --- a/tests/test_search_fusion.py +++ b/tests/test_search_fusion.py @@ -1,6 +1,11 @@ import unittest -from vidxp.application_models import SearchHit, SearchResult +from vidxp.application_models import ( + FusedSearchResult, + RetrievalScoring, + SearchHit, + SearchResult, +) from vidxp.search_fusion import RRF_RANK_CONSTANT, fuse_search_results @@ -109,6 +114,66 @@ def test_rewritten_atomic_query_identity_changes_fused_identity(self): self.assertNotEqual(first.query_id, second.query_id) + def test_fused_result_marks_every_score_ordering_only(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_direction, "higher_is_better") + + def test_fused_result_inherits_the_channel_distance_metric(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + scoring=RetrievalScoring(distance_metric="cosine"), + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.distance_metric, "cosine") + + def test_legacy_stored_result_without_scoring_still_loads(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + # Simulate a job result stored before the scoring descriptor existed. + payload = result.model_dump(mode="json") + payload.pop("scoring") + payload["fusion"].pop("score_direction") + payload["fusion"].pop("score_calibration") + + restored = FusedSearchResult.model_validate(payload) + + self.assertEqual(restored.scoring.distance_metric, "l2") + self.assertEqual(restored.scoring.score_calibration, "ordering_only") + self.assertEqual(restored.fusion.score_calibration, "ordering_only") + if __name__ == "__main__": unittest.main() From 244786a8d73598e0f7d330482345edaa02e5b41e Mon Sep 17 00:00:00 2001 From: AhmedIrfan7 Date: Mon, 14 Sep 2026 18:46:51 +0500 Subject: [PATCH 2/2] fix(search): keep score semantics through query answers and evidence Keep the retrieval distance metric unknown when it was not recorded instead of defaulting to l2. New searches still record the configured metric, and fusion reports a metric only when every searched channel recorded the same one, so an unrecorded channel never becomes a definite metric. The CLI shows "not recorded" in that case. Carry the retrieval descriptor into QueryAnswer so a fused search keeps its metric and score conversion when it becomes a video-query answer, including in stored query job results. Describe evidence scores by their actual source. Search evidence copies the fused moment score, while video-query evidence copies one channel's hit score. EvidenceBoardCandidate, EvidenceBoardTile, and EvidenceDeliveryItem now carry score_semantics: fused_moment with the fusion provenance, or channel_hit with the retrieval descriptor. It is null for actor evidence and for evidence saved before the field existed. All new fields are optional, so stored job results, board requests, and evidence payloads saved earlier still load. --- docs/architecture/platform.md | 22 ++++- src/vidxp/application_models.py | 94 +++++++++++++++++--- src/vidxp/cli_support.py | 2 +- src/vidxp/evidence_delivery.py | 10 +++ src/vidxp/query_service.py | 1 + src/vidxp/search_fusion.py | 18 +++- tests/test_evidence_delivery.py | 146 ++++++++++++++++++++++++++++++++ tests/test_query_service.py | 92 ++++++++++++++++++++ tests/test_search.py | 17 ++++ tests/test_search_fusion.py | 33 +++++++- 10 files changed, 415 insertions(+), 20 deletions(-) diff --git a/docs/architecture/platform.md b/docs/architecture/platform.md index 7b73c565..77c57b55 100644 --- a/docs/architecture/platform.md +++ b/docs/architecture/platform.md @@ -829,12 +829,14 @@ ordering. UI/API/MCP may request stricter limits but cannot loosen policy. Every search response is self-describing about its numbers so callers never have to guess which value they received or which direction ranks better. A `RetrievalScoring` -descriptor is attached to both `SearchResult` and `FusedSearchResult`, and the same -descriptor is returned identically across the CLI, HTTP, MCP, stored job results, and -evidence artifacts. +descriptor is attached to `SearchResult`, `FusedSearchResult`, and `QueryAnswer`, so a +fused search keeps the same meaning when it becomes a video-query answer, and the CLI, +HTTP, MCP, and stored job results all expose it. - `raw_distance` is the raw vector-store distance under `scoring.distance_metric` - (`l2`, `cosine`, or `ip`); smaller is closer. + (`l2`, `cosine`, or `ip`); smaller is closer. The metric is null when it was not + recorded, such as results saved before the descriptor existed, and a fused result + reports it only when every searched channel recorded the same metric. - `score` on a hit is derived as `score = -raw_distance` (`score_transform`), so larger ranks better. - A `SearchHit.rank` is 1-based within one modality channel; ranks from different @@ -847,6 +849,18 @@ evidence artifacts. probability or confidence. Calibrated scores are deferred to the end-to-end ranking evaluation in issue #76. +Evidence items (`EvidenceBoardCandidate`, `EvidenceBoardTile`, and `EvidenceDeliveryItem`) +copy a `score` from their source, and `score_semantics` states which score it is: + +- `fused_moment` for search evidence: the moment's combined reciprocal-rank-fusion + score, described by the carried `fusion` provenance. +- `channel_hit` for video-query moment evidence: one channel's hit score, described by + the carried `scoring` descriptor. Compare it only with hits from the same channel in + the same response. + +`score_semantics` is null when the evidence has no score, such as actor evidence, or +when its source was not recorded. + Actor cluster and detection queries use the same pagination conventions. ## 16. Natural-language query layer diff --git a/src/vidxp/application_models.py b/src/vidxp/application_models.py index acafe60c..64260507 100644 --- a/src/vidxp/application_models.py +++ b/src/vidxp/application_models.py @@ -912,15 +912,19 @@ class RetrievalScoring(ApplicationModel): derived score) and a rank. This descriptor states, in the payload itself, which distance metric produced ``raw_distance``, which direction ranks better, how ``score`` is derived from the distance, and that neither value - is calibrated. It is returned identically across the CLI, HTTP, MCP, - stored job results, and evidence artifacts so the meaning never drifts - between surfaces. Calibrated, probability-like scores are deferred to the - end-to-end ranking evaluation tracked in issue #76. + is calibrated. Search results and video-query answers carry the same + descriptor, and evidence copied from a channel hit carries it too, so the + meaning never drifts between surfaces. Calibrated, probability-like scores are + deferred to the end-to-end ranking evaluation tracked in issue #76. """ - distance_metric: Literal["l2", "cosine", "ip"] = Field( - default="l2", - description="Vector-store distance space that produced each raw_distance.", + distance_metric: Literal["l2", "cosine", "ip"] | None = Field( + default=None, + description=( + "Vector-store distance space that produced each raw_distance. Null " + "when the metric was not recorded, such as results saved before " + "this descriptor existed." + ), ) raw_distance_direction: Literal[RankDirection.lower_is_better] = Field( default=RankDirection.lower_is_better, @@ -981,8 +985,16 @@ class EvidenceBoardCandidate(ApplicationModel): score: float | None = Field( default=None, description=( - "Combined ordering-only fusion score copied from the source moment; " - "larger ranks better, not a probability." + "Ordering-only score copied from the evidence source, never a " + "probability; larger ranks better. score_semantics states whether " + "it is a fused moment score or one channel's hit score." + ), + ) + score_semantics: "EvidenceScoreSemantics | None" = Field( + default=None, + description=( + "Source and meaning of score. Null when the evidence has no score " + "or its source was not recorded." ), ) display_text: str | None = Field(default=None, max_length=512) @@ -994,6 +1006,12 @@ def _valid_interval(self) -> "EvidenceBoardCandidate": raise ValueError("Evidence board candidate end precedes its start.") return self + @model_validator(mode="after") + def _score_semantics_describe_a_score(self) -> "EvidenceBoardCandidate": + if self.score is None and self.score_semantics is not None: + raise ValueError("Evidence score semantics require a score.") + return self + class SearchCommand(ApplicationModel): query: SearchQuery = Field(description="Text to match against indexed moments.") @@ -1153,6 +1171,38 @@ class FusionProvenance(ApplicationModel): ) +class FusedMomentScoreSemantics(ApplicationModel): + """An evidence score copied from a fused moment's combined score.""" + + source: Literal["fused_moment"] = "fused_moment" + fusion: FusionProvenance = Field( + description=( + "Fusion that produced the score: reciprocal-rank fusion over the " + "searched channels, ordering-only and higher-is-better." + ), + ) + + +class ChannelHitScoreSemantics(ApplicationModel): + """An evidence score copied from one channel's search hit.""" + + source: Literal["channel_hit"] = "channel_hit" + scoring: RetrievalScoring = Field( + description=( + "Retrieval descriptor for the hit score: score = -raw_distance, with " + "raw_distance measured under scoring.distance_metric. Ordering-only; " + "compare it only with other hits from the same channel in the same " + "response." + ), + ) + + +EvidenceScoreSemantics = Annotated[ + FusedMomentScoreSemantics | ChannelHitScoreSemantics, + Field(discriminator="source"), +] + + class FusedMoment(ApplicationModel): moment_id: Sha256 | None = None rank: int = Field( @@ -1385,8 +1435,16 @@ class EvidenceDeliveryItem(ApplicationModel): score: float | None = Field( default=None, description=( - "Combined ordering-only fusion score copied from the source moment; " - "larger ranks better, not a probability." + "Ordering-only score copied from the evidence source, never a " + "probability; larger ranks better. score_semantics states whether " + "it is a fused moment score or one channel's hit score." + ), + ) + score_semantics: EvidenceScoreSemantics | None = Field( + default=None, + description=( + "Source and meaning of score. Null when the evidence has no score " + "or its source was not recorded." ), ) provenance: dict[str, JsonValue] = Field(default_factory=dict) @@ -1396,6 +1454,12 @@ class EvidenceDeliveryItem(ApplicationModel): clip: EvidenceArtifact | None = None errors: tuple[ErrorDetail, ...] = () + @model_validator(mode="after") + def _score_semantics_describe_a_score(self) -> "EvidenceDeliveryItem": + if self.score is None and self.score_semantics is not None: + raise ValueError("Evidence score semantics require a score.") + return self + class EvidenceDeliveryResult(ApplicationModel): policy: EvidenceDeliveryPolicy @@ -1488,6 +1552,14 @@ class QueryAnswer(ApplicationModel): model: QueryModelIdentity | None = None claims: tuple[GroundedClaim, ...] = () evidence: tuple[Evidence, ...] = Field(default=(), max_length=200) + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description=( + "Meaning of the rank, score, and raw_distance on the hits in " + "evidence and moments, carried over from the fused search. The " + "combined moment score is described by fusion." + ), + ) moments: tuple[FusedMoment, ...] = () fusion: FusionProvenance evidence_delivery: EvidenceDeliveryResult | None = None diff --git a/src/vidxp/cli_support.py b/src/vidxp/cli_support.py index eadf8f2c..844445cf 100644 --- a/src/vidxp/cli_support.py +++ b/src/vidxp/cli_support.py @@ -152,7 +152,7 @@ def emit_search( caption=( "Rank 1 = best match. Score = reciprocal-rank fusion " "(higher ranks better); ordering-only, not a probability. " - f"Distance metric: {result.scoring.distance_metric}." + f"Distance metric: {result.scoring.distance_metric or 'not recorded'}." ), ) table.add_column("Rank", justify="right") diff --git a/src/vidxp/evidence_delivery.py b/src/vidxp/evidence_delivery.py index ec01cbe3..c25dcff2 100644 --- a/src/vidxp/evidence_delivery.py +++ b/src/vidxp/evidence_delivery.py @@ -5,6 +5,7 @@ from vidxp.application_models import ( ApplicationError, + ChannelHitScoreSemantics, CreateSnippetCommand, ErrorCategory, ErrorDetail, @@ -19,6 +20,7 @@ EvidenceFrameMatch, EvidenceKeyframe, EvidenceRangeResolution, + FusedMomentScoreSemantics, FusedSearchResult, Job, JobKind, @@ -163,6 +165,7 @@ def _search_candidates( result: FusedSearchResult, ) -> tuple[EvidenceBoardCandidate, ...]: candidates: list[EvidenceBoardCandidate] = [] + score_semantics = FusedMomentScoreSemantics(fusion=result.fusion) for moment in result.moments: if moment.moment_id is None: continue @@ -193,6 +196,7 @@ def _search_candidates( else EvidenceFrameMatch.representative ), score=moment.score, + score_semantics=score_semantics, display_text=EvidenceDeliveryService._display_text( selected.metadata ), @@ -217,6 +221,7 @@ def _query_candidates( answer: QueryAnswer, ) -> tuple[EvidenceBoardCandidate, ...]: candidates: list[EvidenceBoardCandidate] = [] + hit_score_semantics = ChannelHitScoreSemantics(scoring=answer.scoring) for rank, evidence in enumerate(answer.evidence, start=1): if isinstance(evidence, MomentEvidence): raw_index = evidence.hit.metadata.get("frame_index") @@ -233,6 +238,7 @@ def _query_candidates( else (evidence.start + evidence.end) / 2 ) score = evidence.hit.score + score_semantics = hit_score_semantics provenance = { "source_id": evidence.source_id, "kind": evidence.kind, @@ -241,6 +247,7 @@ def _query_candidates( frame_index = None representative = (evidence.start + evidence.end) / 2 score = None + score_semantics = None provenance = { "cluster_id": evidence.cluster_id, "detection_count": evidence.detection_count, @@ -264,6 +271,7 @@ def _query_candidates( else EvidenceFrameMatch.representative ), score=score, + score_semantics=score_semantics, display_text=evidence.display_text, provenance=provenance, ) @@ -439,6 +447,7 @@ def _deliver( generation_id=candidate.generation_id, modalities=candidate.modalities, score=candidate.score, + score_semantics=candidate.score_semantics, provenance=candidate.provenance, state=EvidenceDeliveryState.failed, errors=( @@ -525,6 +534,7 @@ def _deliver( generation_id=candidate.generation_id, modalities=candidate.modalities, score=candidate.score, + score_semantics=candidate.score_semantics, provenance=candidate.provenance, state=( EvidenceDeliveryState.partial diff --git a/src/vidxp/query_service.py b/src/vidxp/query_service.py index c8d0aaf0..11ea34ac 100644 --- a/src/vidxp/query_service.py +++ b/src/vidxp/query_service.py @@ -204,6 +204,7 @@ def answer( "plan": plan, "model": self.model.identity if self.model is not None else None, "evidence": evidence, + "scoring": fused.scoring, "moments": fused.moments, "fusion": fused.fusion, } diff --git a/src/vidxp/search_fusion.py b/src/vidxp/search_fusion.py index 13abd7cb..a4c21749 100644 --- a/src/vidxp/search_fusion.py +++ b/src/vidxp/search_fusion.py @@ -101,6 +101,20 @@ def _score(hits: list[SearchHit]) -> float: return sum(1.0 / (RRF_RANK_CONSTANT + rank) for rank in best_ranks.values()) +def _shared_scoring(results: tuple[SearchResult, ...]) -> RetrievalScoring: + """Describe fused hits without turning an unrecorded metric into a known one. + + Channels searched together share one index and normally record the same + metric. The metric is reported only when every channel recorded it + identically; otherwise it stays unknown. + """ + + metrics = {result.scoring.distance_metric for result in results} + return RetrievalScoring( + distance_metric=next(iter(metrics)) if len(metrics) == 1 else None + ) + + def _moment_id( *, snapshot_id: str | None, @@ -215,9 +229,7 @@ def fuse_search_results( ), query=query, modalities=searched_modalities, - scoring=( - ordered_results[0].scoring if ordered_results else RetrievalScoring() - ), + scoring=_shared_scoring(ordered_results), moments=moments, fusion=FusionProvenance( overlap_rule="shared_overlap", diff --git a/tests/test_evidence_delivery.py b/tests/test_evidence_delivery.py index 4c43f112..d672ca29 100644 --- a/tests/test_evidence_delivery.py +++ b/tests/test_evidence_delivery.py @@ -8,9 +8,15 @@ from types import SimpleNamespace from unittest.mock import Mock +from pydantic import ValidationError + from vidxp.application_models import ( + ActorEvidence, ApplicationError, Artifact, + EvidenceBoardCandidate, + EvidenceBoardTile, + EvidenceDeliveryItem, EvidenceDeliveryMode, EvidenceDeliveryPolicy, EvidenceDeliveryState, @@ -22,6 +28,7 @@ QueryAnswer, QueryAnswerMode, QueryPlan, + RetrievalScoring, SearchMomentsPlanStep, SearchHit, SearchResult, @@ -398,6 +405,145 @@ def test_query_delivery_preserves_existing_evidence_identity(self): delivered.evidence[0].evidence_id, ) + def test_search_evidence_score_is_described_as_the_fused_moment_score(self): + service, _artifacts = self.service() + result = service.deliver_search( + fused(), + EvidenceDeliveryPolicy( + mode=EvidenceDeliveryMode.keyframes, + max_items=1, + ), + execution=ExecutionContext(job_id=JOB_ID), + ) + + moment = result.moments[0] + candidate = EvidenceDeliveryService.candidates(result)[0] + item = result.evidence_delivery.items[0] + for evidence in (candidate, item): + self.assertEqual(evidence.score, moment.score) + self.assertEqual(evidence.score_semantics.source, "fused_moment") + self.assertEqual(evidence.score_semantics.fusion, result.fusion) + + def test_query_evidence_score_is_described_as_the_channel_hit_score(self): + service, _artifacts = self.service() + hit = SearchHit.model_validate( + {**scene_hit().model_dump(), "score": -1.0, "raw_distance": 1.0} + ) + atomic = SearchResult( + query_id="scene:known", + query="green frame", + modality="scene", + scoring=RetrievalScoring(distance_metric="cosine"), + hits=(hit,), + ) + fused_result = fuse_search_results( + query="green frame", + requested_modalities=("scene",), + results=(atomic,), + snapshot_id=SNAPSHOT_ID, + ) + answer = QueryAnswer( + question="Which frame is green?", + mode=QueryAnswerMode.evidence_only, + plan=QueryPlan( + steps=(SearchMomentsPlanStep(modality="scene", query="green frame"),) + ), + evidence=( + MomentEvidence( + evidence_id="d" * 64, + snapshot_id=SNAPSHOT_ID, + media_id=MEDIA_ID, + generation_id=GENERATION_ID, + modality="scene", + source_id=hit.source_id, + start=hit.start, + end=hit.end, + hit=hit, + ), + ActorEvidence( + evidence_id="e" * 64, + snapshot_id=SNAPSHOT_ID, + media_id=MEDIA_ID, + generation_id=GENERATION_ID, + cluster_id="cluster-1", + start=1.0, + end=2.0, + detection_count=3, + display_text="Actor cluster cluster-1 appears 3 times.", + ), + ), + scoring=fused_result.scoring, + moments=fused_result.moments, + fusion=fused_result.fusion, + fallback_reason="query_model_not_configured", + ) + + delivered = service.deliver_query( + answer, + EvidenceDeliveryPolicy( + mode=EvidenceDeliveryMode.keyframes, + max_items=2, + ), + execution=ExecutionContext(job_id=JOB_ID), + ) + + # The fused moment score differs from the channel score copied into + # video-query evidence, so the payload must say which one it carries. + self.assertAlmostEqual(fused_result.moments[0].score, 1 / 61) + moment_candidate, actor_candidate = EvidenceDeliveryService.candidates( + delivered + ) + moment_item, actor_item = delivered.evidence_delivery.items + for evidence in (moment_candidate, moment_item): + self.assertEqual(evidence.score, -1.0) + self.assertEqual(evidence.score_semantics.source, "channel_hit") + scoring = evidence.score_semantics.scoring + self.assertEqual(scoring.distance_metric, "cosine") + self.assertEqual(scoring.score_transform, "negated_distance") + for evidence in (actor_candidate, actor_item): + self.assertIsNone(evidence.score) + self.assertIsNone(evidence.score_semantics) + + def test_evidence_score_semantics_require_a_score(self): + candidate = EvidenceDeliveryService.candidates(fused())[0] + + with self.assertRaises(ValidationError): + EvidenceBoardCandidate.model_validate( + {**candidate.model_dump(), "score": None} + ) + with self.assertRaises(ValidationError): + EvidenceDeliveryItem( + evidence_id=candidate.evidence_id, + rank=1, + media_id=MEDIA_ID, + generation_id=GENERATION_ID, + modalities=("scene",), + score=None, + score_semantics=candidate.score_semantics, + state=EvidenceDeliveryState.ready, + ) + + def test_board_tiles_keep_score_semantics_and_legacy_evidence_loads(self): + candidate = EvidenceDeliveryService.candidates(fused())[0] + + # Board tiles are built from candidates the same way the board job does. + tile = EvidenceBoardTile( + **candidate.model_dump(), + tile_id="f" * 64, + page_number=1, + position=1, + state=EvidenceDeliveryState.ready, + ) + self.assertEqual(tile.score_semantics, candidate.score_semantics) + + # A candidate stored before score semantics existed still loads, and its + # source stays unrecorded instead of being guessed. + legacy = candidate.model_dump(mode="json") + legacy.pop("score_semantics") + restored = EvidenceBoardCandidate.model_validate(legacy) + self.assertEqual(restored.score, candidate.score) + self.assertIsNone(restored.score_semantics) + @unittest.skipUnless(shutil.which("ffmpeg"), "ffmpeg is required") class ExactFrameRendererTests(unittest.TestCase): diff --git a/tests/test_query_service.py b/tests/test_query_service.py index e1fa3cd9..b532d393 100644 --- a/tests/test_query_service.py +++ b/tests/test_query_service.py @@ -6,14 +6,18 @@ FusedSearchResult, FusionProvenance, IndexSnapshotReference, + QueryAnswer, QueryAnswerMode, QueryModelIdentity, QueryPlan, QueryVideoCommand, + RetrievalScoring, SearchHit, SearchMomentsPlanStep, SearchResult, ) +from vidxp.capabilities.search import search_embeddings +from vidxp.core.contracts import IndexConfig from vidxp.ports import QueryProviderError from vidxp.query_service import GroundedQueryService from vidxp.search_fusion import fuse_search_results @@ -372,6 +376,94 @@ def test_evidence_is_bounded_to_retained_fused_moments(self): all(item.source_id in retained_sources for item in evidence) ) + def test_query_answer_keeps_cosine_retrieval_scoring(self): + class CosineStore: + def query(self, modality, embedding, **options): + return [ + { + "source_id": "speech:1", + "raw_distance": 0.25, + "metadata": { + "video_id": MEDIA_ID, + "generation_id": GENERATION_ID, + "start": 1.0, + "end": 2.0, + "text": "the taxi arrived", + }, + } + ] + + config = IndexConfig( + dataset="sample", + split="test", + run_id="run-1", + enabled_modalities=("speech",), + vector_distance="cosine", + ) + atomic = search_embeddings( + "taxi", + "speech", + [0.5, 0.25], + config=config, + required_metadata=frozenset({"text"}), + storage=CosineStore(), + ) + fused_result = fused("speech", atomic) + service = GroundedQueryService() + evidence = service.evidence( + snapshot=self.snapshot, + fused=fused_result, + actors=(), + ) + + answer = service.answer( + self.command, + plan=QueryPlan( + steps=(SearchMomentsPlanStep(modality="speech", query="taxi"),) + ), + planning_fallback="query_model_not_configured", + evidence=evidence, + fused=fused_result, + ) + + self.assertEqual(answer.scoring, fused_result.scoring) + self.assertEqual(answer.scoring.distance_metric, "cosine") + self.assertEqual(answer.scoring.score_transform, "negated_distance") + hit = answer.evidence[0].hit + self.assertEqual(hit.score, -hit.raw_distance) + # Stored query job results keep the descriptor too. + restored = QueryAnswer.model_validate_json(answer.model_dump_json()) + self.assertEqual(restored.scoring.distance_metric, "cosine") + + def test_legacy_query_answer_without_scoring_keeps_metric_unknown(self): + atomic = result(text="the taxi arrived", modality="speech").model_copy( + update={"scoring": RetrievalScoring(distance_metric="cosine")} + ) + fused_result = fused("speech", atomic) + service = GroundedQueryService() + evidence = service.evidence( + snapshot=self.snapshot, + fused=fused_result, + actors=(), + ) + answer = service.answer( + self.command, + plan=QueryPlan( + steps=(SearchMomentsPlanStep(modality="speech", query="taxi"),) + ), + planning_fallback="query_model_not_configured", + evidence=evidence, + fused=fused_result, + ) + self.assertEqual(answer.scoring.distance_metric, "cosine") + + # Simulate a query job result stored before the descriptor existed. + payload = answer.model_dump(mode="json") + payload.pop("scoring") + restored = QueryAnswer.model_validate(payload) + + self.assertIsNone(restored.scoring.distance_metric) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_search.py b/tests/test_search.py index 6b43595f..2eedb98f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -141,6 +141,23 @@ def test_result_describes_its_metric_and_ordering_only_scores(self): self.assertEqual(scoring.score_calibration, "ordering_only") self.assertEqual(scoring.hit_rank_direction, "lower_is_better") + def test_default_distance_metric_is_recorded_explicitly(self): + storage = FakeStorage([dialogue_row("run:video-1:dialogue:a", 0.1)]) + with patch( + "vidxp.capabilities.speech.operations.speech_embedding", + return_value=[0.5, 0.25], + ): + result = search_speech( + "fresh bread", + config=self.config, + runtime=self.runtime, + top_k=1, + storage=storage, + ) + + self.assertEqual(self.config.vector_distance, "l2") + self.assertEqual(result.scoring.distance_metric, "l2") + def test_dialogue_query_uses_model_owned_query_prompt(self): encoder = Mock() encoder.encode_query.return_value = np.asarray([[0.5, 0.25]]) diff --git a/tests/test_search_fusion.py b/tests/test_search_fusion.py index 0664ccd7..d6495d6f 100644 --- a/tests/test_search_fusion.py +++ b/tests/test_search_fusion.py @@ -323,10 +323,41 @@ def test_legacy_stored_result_without_scoring_still_loads(self): restored = FusedSearchResult.model_validate(payload) - self.assertEqual(restored.scoring.distance_metric, "l2") + # The metric was never recorded, so it must stay unknown rather than + # silently becoming a definite metric. + self.assertIsNone(restored.scoring.distance_metric) self.assertEqual(restored.scoring.score_calibration, "ordering_only") self.assertEqual(restored.fusion.score_calibration, "ordering_only") + def test_fused_metric_is_unknown_unless_every_channel_agrees(self): + def channel(modality, metric): + return SearchResult( + query_id=f"{modality}:q", + query="taxi", + modality=modality, + scoring=RetrievalScoring(distance_metric=metric), + hits=(hit(modality, 1, 1, 2, f"{modality}:1"),), + ) + + def fused_metric(*results): + return fuse_search_results( + query="taxi", + requested_modalities=tuple(result.modality for result in results), + results=results, + ).scoring.distance_metric + + self.assertEqual( + fused_metric(channel("scene", "cosine"), channel("speech", "cosine")), + "cosine", + ) + self.assertIsNone( + fused_metric(channel("scene", "cosine"), channel("speech", None)) + ) + self.assertIsNone( + fused_metric(channel("scene", "cosine"), channel("speech", "ip")) + ) + self.assertIsNone(fused_metric()) + if __name__ == "__main__": unittest.main()