From 98c6069250b6fcc52922b96ee8c04b2f1836f1a5 Mon Sep 17 00:00:00 2001 From: Sidra-009 Date: Sat, 5 Sep 2026 08:32:16 +0500 Subject: [PATCH 1/2] feat(query): add temporal_relation and evidence_request plan steps --- pyproject.toml | 5 +++ src/vidxp/application_models.py | 34 +++++++++++++++++++- src/vidxp/query_service.py | 26 ++++++++++++++- uv.lock | 56 +++++++++++++++++++-------------- 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ee89112..9e0ad308 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,3 +161,8 @@ torchaudio = [ name = "pytorch-cpu" url = "https://download.pytorch.org/whl/cpu" explicit = true + +[dependency-groups] +dev = [ + "pytest>=9.1.1", +] diff --git a/src/vidxp/application_models.py b/src/vidxp/application_models.py index 395bde42..8a64ef41 100644 --- a/src/vidxp/application_models.py +++ b/src/vidxp/application_models.py @@ -1044,18 +1044,48 @@ def _unique_modalities( return values +class OccurrenceMode(StrEnum): + first = "first" + best = "best" + all = "all" + + class SearchMomentsPlanStep(ApplicationModel): kind: Literal["search_moments"] = "search_moments" modality: Identifier query: SearchQuery + occurrence_mode: OccurrenceMode = OccurrenceMode.best class ActorOverviewPlanStep(ApplicationModel): kind: Literal["actor_overview"] = "actor_overview" +class TemporalRelation(StrEnum): + before = "before" + after = "after" + during = "during" + + +class TemporalRelationPlanStep(ApplicationModel): + kind: Literal["temporal_relation"] = "temporal_relation" + relation: TemporalRelation + reference_query: SearchQuery + target_modality: Identifier + target_query: SearchQuery + + +class EvidenceRequestPlanStep(ApplicationModel): + kind: Literal["evidence_request"] = "evidence_request" + delivery_mode: EvidenceDeliveryMode + include_board: bool = False + + QueryPlanStep = Annotated[ - SearchMomentsPlanStep | ActorOverviewPlanStep, + SearchMomentsPlanStep + | ActorOverviewPlanStep + | TemporalRelationPlanStep + | EvidenceRequestPlanStep, Field(discriminator="kind"), ] @@ -1068,6 +1098,8 @@ class QueryPlanningRequest(ApplicationModel): question: SearchQuery allowed_modalities: tuple[Identifier, ...] actor_overview_allowed: bool = False + temporal_relations_allowed: bool = False + evidence_requests_allowed: bool = False class QueryModelIdentity(ApplicationModel): diff --git a/src/vidxp/query_service.py b/src/vidxp/query_service.py index c8d0aaf0..09a647dd 100644 --- a/src/vidxp/query_service.py +++ b/src/vidxp/query_service.py @@ -5,8 +5,10 @@ from vidxp.application_models import ( ActorEvidence, + ActorOverviewPlanStep, DraftAnswer, Evidence, + EvidenceRequestPlanStep, FusedSearchResult, GroundedClaim, IndexSnapshotReference, @@ -18,7 +20,7 @@ QuerySynthesisRequest, QueryVideoCommand, SearchMomentsPlanStep, - ActorOverviewPlanStep, + TemporalRelationPlanStep, ) from vidxp.capabilities.actor.schemas import ActorClusterSummary from vidxp.ports import QueryModelPort, QueryProviderError @@ -39,6 +41,8 @@ def _default_plan( *, search_modalities: tuple[str, ...], actor_overview: bool, + temporal_relations: bool = False, + evidence_requests: bool = False, ) -> QueryPlan: steps = [ SearchMomentsPlanStep( @@ -57,6 +61,8 @@ def _valid_plan( *, search_modalities: tuple[str, ...], actor_overview: bool, + temporal_relations: bool = False, + evidence_requests: bool = False, ) -> bool: searches = [ step.modality @@ -66,6 +72,16 @@ def _valid_plan( actor_steps = sum( isinstance(step, ActorOverviewPlanStep) for step in plan.steps ) + temporal_steps = sum( + isinstance(step, TemporalRelationPlanStep) for step in plan.steps + ) + evidence_steps = sum( + isinstance(step, EvidenceRequestPlanStep) for step in plan.steps + ) + if not temporal_relations and temporal_steps > 0: + return False + if not evidence_requests and evidence_steps > 0: + return False return ( len(searches) == len(set(searches)) and set(searches) == set(search_modalities) @@ -85,11 +101,15 @@ def plan( *, search_modalities: tuple[str, ...], actor_overview: bool, + temporal_relations: bool = False, + evidence_requests: bool = False, ) -> tuple[QueryPlan, str | None]: fallback = _default_plan( command, search_modalities=search_modalities, actor_overview=actor_overview, + temporal_relations=temporal_relations, + evidence_requests=evidence_requests, ) if self.model is None: return fallback, "query_model_not_configured" @@ -99,6 +119,8 @@ def plan( question=command.question, allowed_modalities=search_modalities, actor_overview_allowed=actor_overview, + temporal_relations_allowed=temporal_relations, + evidence_requests_allowed=evidence_requests, ) ) except QueryProviderError: @@ -107,6 +129,8 @@ def plan( proposed, search_modalities=search_modalities, actor_overview=actor_overview, + temporal_relations=temporal_relations, + evidence_requests=evidence_requests, ): return fallback, "query_plan_rejected" return proposed, None diff --git a/uv.lock b/uv.lock index 159418d3..0ce9e523 100644 --- a/uv.lock +++ b/uv.lock @@ -3773,7 +3773,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3852,7 +3852,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -4233,13 +4233,13 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "fsspec", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "jinja2", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "networkx", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "setuptools", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "sympy", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "typing-extensions", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/5b/fe/cba54dc58523434919b66f13a667e36e436deddd77ca519e96553617d4ec/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", size = 111187938, upload-time = "2026-07-08T16:05:17.065Z" }, @@ -4260,13 +4260,13 @@ resolution-markers = [ "(python_full_version < '3.12' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform == 'win32')", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "fsspec", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "jinja2", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "networkx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "setuptools", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "sympy", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0%2Bcpu-cp311-cp311-linux_s390x.whl", hash = "sha256:6e9817dbdf5ea76789babd46e457eac5bf14ff566cf85f8addbfdff2d56601ce", upload-time = "2026-07-08T19:27:52Z" }, @@ -4355,10 +4355,10 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pillow" }, - { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "pillow", marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "torch", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/7a/b2/1e010052079e4c577007b789db336ea7075f1a426e84d17121fbc3745516/torchvision-0.28.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:83fe6c020866a85acd7d97deccc45ff11d66daf42916d04396a4309c66c0ccb8", size = 1856017, upload-time = "2026-07-08T16:07:55.533Z" }, @@ -4379,10 +4379,10 @@ resolution-markers = [ "(python_full_version < '3.12' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform == 'win32')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pillow" }, - { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform == 'win32')" }, + { name = "pillow", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.28.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:22958193d72444ed7cbcc665ba4821a31e5279f9c4d1ad08520918b30896b78a", upload-time = "2026-07-08T12:26:39Z" }, @@ -4739,6 +4739,11 @@ test = [ { name = "pytest" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "alembic", marker = "extra == 'server'", specifier = ">=1.18.5,<2" }, @@ -4878,6 +4883,9 @@ requires-dist = [ ] provides-extras = ["storage", "speech", "scene", "actor", "action", "sound", "all", "local-worker", "mcp", "slm", "server", "server-worker", "test", "frontend", "benchmarks"] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.1.1" }] + [[package]] name = "watchdog" version = "6.0.0" From 60b4ebae9885ab7aac9383c847447c3a68bf9f71 Mon Sep 17 00:00:00 2001 From: Sidra-009 Date: Wed, 16 Sep 2026 20:45:36 +0500 Subject: [PATCH 2/2] fix(query): reject unsupported occurrence modes and validate temporal target_modality - Reject first/all occurrence modes, fall back to ranked search since execution doesn't implement them yet - Validate TemporalRelationPlanStep.target_modality against search_modalities, reject unknown modalities with existing fallback - Add test coverage for both rejection paths - Remove unrelated pytest dev-group from pyproject.toml/uv.lock Addresses review comments from @tulayha on #147 --- pyproject.toml | 5 --- src/vidxp/query_service.py | 12 ++++++ tests/test_query_service.py | 77 +++++++++++++++++++++++++++++++++++++ uv.lock | 8 ---- 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9e0ad308..9ee89112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,8 +161,3 @@ torchaudio = [ name = "pytorch-cpu" url = "https://download.pytorch.org/whl/cpu" explicit = true - -[dependency-groups] -dev = [ - "pytest>=9.1.1", -] diff --git a/src/vidxp/query_service.py b/src/vidxp/query_service.py index 09a647dd..8c640727 100644 --- a/src/vidxp/query_service.py +++ b/src/vidxp/query_service.py @@ -78,10 +78,22 @@ def _valid_plan( evidence_steps = sum( isinstance(step, EvidenceRequestPlanStep) for step in plan.steps ) + if any( + step.occurrence_mode.value != "best" + for step in plan.steps + if isinstance(step, SearchMomentsPlanStep) + ): + return False if not temporal_relations and temporal_steps > 0: return False if not evidence_requests and evidence_steps > 0: return False + if any( + step.target_modality not in search_modalities + for step in plan.steps + if isinstance(step, TemporalRelationPlanStep) + ): + return False return ( len(searches) == len(set(searches)) and set(searches) == set(search_modalities) diff --git a/tests/test_query_service.py b/tests/test_query_service.py index e1fa3cd9..7d225112 100644 --- a/tests/test_query_service.py +++ b/tests/test_query_service.py @@ -13,6 +13,7 @@ SearchHit, SearchMomentsPlanStep, SearchResult, + TemporalRelationPlanStep, ) from vidxp.ports import QueryProviderError from vidxp.query_service import GroundedQueryService @@ -121,6 +122,82 @@ def test_invalid_model_plan_falls_back_to_complete_closed_plan(self): ["scene", "speech"], ) + def test_best_occurrence_mode_is_supported(self): + model = FakeQueryModel( + QueryPlan( + steps=( + SearchMomentsPlanStep( + modality="scene", + query="taxi", + occurrence_mode="best", + ), + ) + ) + ) + service = GroundedQueryService(model) + + plan, reason = service.plan( + self.command, + search_modalities=("scene",), + actor_overview=False, + ) + + self.assertIsNone(reason) + self.assertEqual(plan.steps[0].occurrence_mode.value, "best") + + def test_unsupported_occurrence_mode_falls_back_to_ranked_search(self): + for occurrence_mode in ("first", "all"): + model = FakeQueryModel( + QueryPlan( + steps=( + SearchMomentsPlanStep( + modality="scene", + query="taxi", + occurrence_mode=occurrence_mode, + ), + ) + ) + ) + service = GroundedQueryService(model) + + plan, reason = service.plan( + self.command, + search_modalities=("scene",), + actor_overview=False, + ) + + self.assertEqual(reason, "query_plan_rejected") + self.assertEqual(plan.steps[0].occurrence_mode.value, "best") + + def test_unknown_temporal_target_modality_falls_back(self): + model = FakeQueryModel( + QueryPlan( + steps=( + SearchMomentsPlanStep( + modality="scene", + query="taxi", + ), + TemporalRelationPlanStep( + relation="before", + reference_query="taxi", + target_modality="nonexistent", + target_query="car", + ), + ) + ) + ) + service = GroundedQueryService(model) + + plan, reason = service.plan( + self.command, + search_modalities=("scene",), + actor_overview=False, + temporal_relations=True, + ) + + self.assertEqual(reason, "query_plan_rejected") + self.assertEqual([step.modality for step in plan.steps], ["scene"]) + def test_provider_failure_uses_deterministic_retrieval_plan(self): service = GroundedQueryService( FakeQueryModel(QueryProviderError("offline")) diff --git a/uv.lock b/uv.lock index 0ce9e523..7b3f713b 100644 --- a/uv.lock +++ b/uv.lock @@ -4739,11 +4739,6 @@ test = [ { name = "pytest" }, ] -[package.dev-dependencies] -dev = [ - { name = "pytest" }, -] - [package.metadata] requires-dist = [ { name = "alembic", marker = "extra == 'server'", specifier = ">=1.18.5,<2" }, @@ -4883,9 +4878,6 @@ requires-dist = [ ] provides-extras = ["storage", "speech", "scene", "actor", "action", "sound", "all", "local-worker", "mcp", "slm", "server", "server-worker", "test", "frontend", "benchmarks"] -[package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=9.1.1" }] - [[package]] name = "watchdog" version = "6.0.0"