diff --git a/src/vidxp/capabilities/actor/indexing.py b/src/vidxp/capabilities/actor/indexing.py index 35da06c2..e53329ca 100644 --- a/src/vidxp/capabilities/actor/indexing.py +++ b/src/vidxp/capabilities/actor/indexing.py @@ -74,7 +74,7 @@ def _actor_records( records.append( StorageRecord( source_id=source_id, - embedding=[0.0], + embedding=detection["encoding"], metadata={ **config.record_identity("actor", source_id), "detection_id": detection["detection_id"], @@ -94,6 +94,7 @@ def _actor_records( def _actor_cluster_records( cluster_sizes: dict[str, int], cluster_ranges: dict[str, tuple[float, float]], + cluster_embeddings: dict[str, Any], config: IndexConfig, ) -> list[StorageRecord]: records = [] @@ -110,7 +111,7 @@ def _actor_cluster_records( records.append( StorageRecord( source_id=source_id, - embedding=[0.0], + embedding=[float(value) for value in cluster_embeddings[cluster_id]], metadata={ **config.record_identity("actor", source_id), "record_kind": "cluster_summary", @@ -209,6 +210,7 @@ def process_actor_samples( min(height, int(face[1] + face[3])), max(0, int(face[0])), ), + "encoding": encoding.tolist(), } ) state.processed_frames += 1 @@ -251,6 +253,11 @@ def finalize_actor_index( cluster_id: state.cluster_ranges[cluster_id] for cluster_id in retained }, + { + cluster_id: state.known_encodings[index] + for index, cluster_id in enumerate(state.known_ids) + if cluster_id in retained + }, config, ), batch_size=config.storage_batch_size, diff --git a/src/vidxp/core/contracts.py b/src/vidxp/core/contracts.py index c58d632f..34162bb8 100644 --- a/src/vidxp/core/contracts.py +++ b/src/vidxp/core/contracts.py @@ -11,7 +11,7 @@ from urllib.parse import quote -INDEX_SCHEMA_VERSION = 7 +INDEX_SCHEMA_VERSION = 8 MANIFEST_SCHEMA_VERSION = 2 diff --git a/tests/test_indexing.py b/tests/test_indexing.py index b60b2fb4..0ae144e2 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -444,6 +444,7 @@ def test_actor_records_preserve_stable_detection_metadata(self): "frame_index": 0, "timestamp": 0.0, "bbox": (1, 2, 3, 0), + "encoding": [0.6, 0.8], } ], config, @@ -455,6 +456,73 @@ def test_actor_records_preserve_stable_detection_metadata(self): "generation-1:actors:video-1:actor-cluster:1", ) self.assertEqual(records[0].metadata["bbox_top"], 1) + self.assertEqual(records[0].embedding, [0.6, 0.8]) + + def test_actor_indexing_persists_real_face_embeddings_not_placeholders(self): + import numpy as np + from unittest.mock import Mock + + from vidxp.capabilities.actor.indexing import ( + ActorIndexState, + process_actor_samples, + ) + from vidxp.core.video import FrameSample + + config = IndexConfig( + dataset="sample", + split="test", + run_id="actors", + video_id="video-1", + generation_id="generation-1", + enabled_modalities=("actor",), + ) + + raw_encoding = np.array([3.0, 4.0], dtype="float32") + expected_normalized = (raw_encoding / np.linalg.norm(raw_encoding)).tolist() + + detector = Mock() + detector.setInputSize = Mock() + detector.setScoreThreshold = Mock() + detector.detect.return_value = ( + True, + np.array([[10.0, 10.0, 20.0, 20.0]], dtype="float32"), + ) + + recognizer = Mock() + recognizer.alignCrop.return_value = np.zeros((112, 112, 3), dtype="uint8") + recognizer.feature.return_value = raw_encoding.reshape(1, -1) + + state = ActorIndexState( + models=Mock(detector=detector, recognizer=recognizer) + ) + storage = Mock() + + samples = [ + FrameSample( + frame_index=0, + timestamp=0.0, + frame=np.zeros((48, 48, 3), dtype="uint8"), + ), + ] + + process_actor_samples( + samples, + state=state, + config=config, + storage=storage, + cancellation=CancellationToken(), + ) + + self.assertEqual(storage.upsert.call_count, 1) + (_, records), _ = storage.upsert.call_args + self.assertEqual(len(records), 1) + record = records[0] + + self.assertIsNotNone(record.embedding) + self.assertNotEqual(list(record.embedding), [0.0]) + self.assertEqual(len(record.embedding), 2) + for actual, expected in zip(record.embedding, expected_normalized): + self.assertAlmostEqual(actual, expected, places=5) def test_actor_cluster_identity_is_unique_by_media_and_generation(self): def config(video_id, generation_id): @@ -484,6 +552,7 @@ def test_actor_cluster_summaries_are_materialized_for_bounded_paging(self): records = _actor_cluster_records( {"cluster-1": 3}, {"cluster-1": (1.25, 4.5)}, + {"cluster-1": [0.6, 0.8]}, config, ) @@ -496,6 +565,7 @@ def test_actor_cluster_summaries_are_materialized_for_bounded_paging(self): self.assertEqual(records[0].metadata["detection_count"], 3) self.assertEqual(records[0].metadata["first_timestamp"], 1.25) self.assertEqual(records[0].metadata["last_timestamp"], 4.5) + self.assertEqual(records[0].embedding, [0.6, 0.8]) if __name__ == "__main__": diff --git a/tests/test_storage_integration.py b/tests/test_storage_integration.py index 6bdc7a3a..03c31e2a 100644 --- a/tests/test_storage_integration.py +++ b/tests/test_storage_integration.py @@ -8,6 +8,8 @@ StorageRecord, ) from vidxp.core.storage import IndexStorage, metadata_filter +from vidxp.capabilities.actor.config import actor_config +from vidxp.capabilities.actor.indexing import ActorIndexState, finalize_actor_index class ChromaStorageIntegrationTests(unittest.TestCase): @@ -130,6 +132,84 @@ def test_generation_scope_and_cleanup_use_chroma_in_filter(self): ["generation-2"], ) + + def test_actor_finalization_persists_cluster_summary_with_matching_embedding_dimension(self): + import numpy as np + from unittest.mock import Mock + + with TemporaryDirectory() as directory: + config = IndexConfig( + dataset="sample", + split="test", + run_id="actors", + video_id="video-1", + generation_id="generation-1", + enabled_modalities=("actor",), + storage_directory=directory, + ) + + centroid = np.zeros(128, dtype="float32") + centroid[0] = 1.0 + + cluster_id = "generation-1:actors:video-1:actor-cluster:1" + + state = ActorIndexState( + models=Mock(), + known_ids=[cluster_id], + known_encodings=[centroid], + cluster_sizes={cluster_id: 4}, + cluster_ranges={cluster_id: (1.0, 3.0)}, + ) + + with IndexStorage(config) as storage: + storage.upsert( + "actor", + [ + StorageRecord( + source_id="detection-1", + embedding=centroid.tolist(), + metadata={ + **config.record_identity( + "actor", "detection-1" + ), + "detection_id": "detection-1", + "cluster_id": cluster_id, + "frame_index": 0, + "timestamp": 1.0, + "bbox_top": 0, + "bbox_right": 10, + "bbox_bottom": 10, + "bbox_left": 0, + }, + ) + ], + batch_size=1, + cancellation=CancellationToken(), + ) + + finalize_actor_index( + state, + config=config, + storage=storage, + ) + + result = storage.collection("actor").get( + include=["embeddings", "metadatas"], + ) + + summary_embeddings = [ + embedding + for embedding, metadata in zip( + result["embeddings"], + result["metadatas"], + ) + if metadata.get("record_kind") == "cluster_summary" + ] + + self.assertEqual(len(summary_embeddings), 1) + self.assertEqual(len(summary_embeddings[0]), 128) + self.assertEqual(list(summary_embeddings[0]), centroid.tolist()) + def test_read_only_store_fails_closed_without_database_or_collection(self): with TemporaryDirectory() as directory: path = Path(directory)