fix(index): recover an index an incompatible schema wrote

INSTALLATION_GUIDE.md

@@ -270,6 +270,23 @@ use the same indexing settings. Bulk indexing does not migrate an existing
 multi-video index to a different profile or remove old results to make room
 for one. Such jobs report the existing profile-compatibility error.
 
+### Rebuild an index VidXP can no longer use
+
+If an upgrade changes how indexes are stored, VidXP reports that the existing
+index is incompatible. Indexing, clearing, and even reading the index status
+fail while that data is in place. Discard the generated index data and start
+over:
+
+```bash
+vidxp index reset --yes
+vidxp index bulk --all --modality scene
+```
+
+Resetting deletes every generated index in the repository, including its
+vector collections, and reports which videos it gave up. Imported media is
+preserved, so the second command rebuilds search from the videos already in
+the repository. Nothing has to be imported again.
+
 ### Start an installed interface
 
 | Interface | Command |

src/vidxp/application.py

@@ -34,6 +34,7 @@
     PrepareModelsCommand,
     PrepareModelsResult,
     RemoveIndexCommand,
+    IndexResetResult,
     ResourceNotFoundError,
     RuntimeReadiness,
     QueryAnswer,
@@ -938,6 +939,26 @@ def clear_index(self) -> bool:
                 self.registry.install_hint(self.registry.index_names()),
             ) from exc
 
+    @application_boundary
+    def reset_index(self) -> IndexResetResult:
+        """Discard generated index data so the repository can be rebuilt.
+
+        Use this when an index cannot be cleared or rebuilt because its
+        generations were written by an incompatible index schema. Imported
+        media is preserved, so indexing it again restores search.
+        """
+
+        base_config = self._base_config()
+        try:
+            return IndexResetResult.model_validate(
+                self.index_backend.reset(base_config)
+            )
+        except ModuleNotFoundError as exc:
+            raise DependencyUnavailableError(
+                self.registry.index_names(),
+                self.registry.install_hint(self.registry.index_names()),
+            ) from exc
+
     @application_boundary
     def remove_from_index(self, command: RemoveIndexCommand) -> bool:
         return self.index_backend.remove(

src/vidxp/application_boundary.py

@@ -97,7 +97,10 @@ def wrapped(*args: Any, **kwargs: Any) -> Any:
             raise ApplicationError(
                 "index_schema_incompatible",
                 ErrorCategory.conflict,
-                "The index schema is incompatible with this version.",
+                "The index schema is incompatible with this version. "
+                "Discard the generated index data with "
+                "`vidxp index reset --yes`, then index the media again. "
+                "Imported media is preserved.",
             ) from exc
         except IndexStorageUnavailableError as exc:
             raise ApplicationError(

src/vidxp/application_models.py

@@ -725,6 +725,35 @@ def skipped(self) -> tuple[BulkIndexTarget, ...]:
         )
 
 
+class IndexResetResult(ApplicationModel):
+    """What discarding generated index data gave up.
+
+    Imported media is never part of a reset, so it is not reported here.
+    """
+
+    discarded: bool = Field(
+        description=(
+            "Whether an active snapshot with generated data was replaced."
+        )
+    )
+    media_ids: tuple[MediaId, ...] = Field(
+        default=(),
+        description="Media that must be indexed again to become searchable.",
+    )
+    generation_ids: tuple[IndexGenerationId, ...] = Field(
+        default=(),
+        description="Generations the discarded snapshot referenced.",
+    )
+    modalities: tuple[str, ...] = Field(
+        default=(),
+        description="Capabilities the discarded snapshot had indexed.",
+    )
+    collections: tuple[str, ...] = Field(
+        default=(),
+        description="Vector collections deleted so they can be rebuilt.",
+    )
+
+
 class RemoveIndexCommand(ApplicationModel):
     media_id: MediaId
 

src/vidxp/cli_commands/index.py

@@ -466,6 +466,70 @@ def index_list(
         )
 
 
+@app.command("reset")
+def index_reset(
+    ctx: typer.Context,
+    yes: Annotated[
+        bool,
+        typer.Option("--yes", "-y", help="Skip the confirmation prompt."),
+    ] = False,
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Emit machine-readable JSON."),
+    ] = False,
+) -> None:
+    """Discard generated index data so an unusable index can be rebuilt.
+
+    Use this when indexing or clearing fails because the existing index was
+    written by an incompatible index schema. Imported media is preserved, so
+    indexing it again restores search.
+    """
+
+    state = state_from_context(ctx)
+    output_format = effective_output_format(state, json_output)
+    if not yes:
+        typer.secho(
+            "Resetting discards every generated index in this repository, "
+            "including its vector collections.",
+            fg=typer.colors.YELLOW,
+        )
+        typer.echo(
+            "Imported media is preserved. Indexed media has to be indexed "
+            "again before it can be searched."
+        )
+        typer.confirm(
+            f"Discard the generated index data at "
+            f"{state.service.index_directory}?",
+            abort=True,
+        )
+    result = state.service.reset_index()
+    payload = result.model_dump(mode="json")
+    if output_format == OutputFormat.json:
+        emit_json(payload)
+        return
+    if not result.discarded and not result.collections:
+        typer.echo("No generated index data was found.")
+        return
+    if result.media_ids:
+        table = Table(title="Discarded index data")
+        table.add_column("Media ID")
+        table.add_column("Generation")
+        for media_id, generation_id in zip(
+            result.media_ids,
+            result.generation_ids,
+        ):
+            table.add_row(media_id, generation_id)
+        Console().print(table)
+    typer.secho("Generated index data discarded.", fg=typer.colors.GREEN)
+    typer.echo(
+        f"Media needing indexing again: {len(result.media_ids)}; "
+        f"capabilities affected: {', '.join(result.modalities) or 'none'}; "
+        f"vector collections deleted: "
+        f"{', '.join(result.collections) or 'none'}."
+    )
+    typer.echo("Rebuild with: vidxp index bulk")
+
+
 @app.command("clear")
 def index_clear(
     ctx: typer.Context,

src/vidxp/core/storage.py

@@ -231,6 +231,24 @@ def clear(self, modalities: Iterable[str] | None = None) -> None:
                 self._call(self.client.delete_collection, name)
             self._collections.pop(modality, None)
 
+    def drop_all_collections(self) -> tuple[str, ...]:
+        """Delete every collection in this store and report what was dropped.
+
+        Recovery has to remove collections this configuration does not name.
+        An incompatible store can hold collections written by another index
+        profile, and a collection keeps the embedding dimensions it was
+        created with, so reusing one after a schema change fails.
+        """
+
+        dropped = sorted(
+            getattr(collection, "name", collection)
+            for collection in self._call(self.client.list_collections)
+        )
+        for name in dropped:
+            self._call(self.client.delete_collection, name)
+        self._collections.clear()
+        return tuple(dropped)
+
     def delete_video(self, modality: str, video_id: str) -> None:
         self._call(
             self.collection(modality).delete,

src/vidxp/infrastructure/local_index.py

@@ -583,3 +583,41 @@ def clear(self, config: IndexConfig) -> bool:
         repository = self.repository
         with repository.lease():
             return repository.clear()
+
+    def reset(self, config: IndexConfig) -> dict[str, object]:
+        """Discard generated index data so the repository can be rebuilt.
+
+        Clearing validates the active generations first, so it cannot recover
+        a repository whose generations were written by an incompatible index
+        schema. Reset reads the snapshot document without those generations
+        and drops the vector collections as well, because a collection keeps
+        the embedding dimensions it was created with.
+
+        Imported media is untouched. Retained generation directories are left
+        on disk, as `clear` leaves them.
+        """
+
+        self._require_index_directory(config.index_directory)
+        repository = self.repository
+        with repository.lease():
+            summary = repository.describe_active_generations()
+            dropped: tuple[str, ...] = ()
+            store_config = replace(
+                config,
+                storage_directory=repository.store,
+            )
+            # Drop the vector collections before replacing the metadata. A
+            # failure here then leaves the repository exactly as it was, so
+            # the operation can be retried and still report what it removed.
+            if store_config.index_directory.is_dir():
+                # Opened for writing rather than through the committed-store
+                # helper: that helper rejects a store holding no collections,
+                # and a repository with nothing left to drop still has to be
+                # recoverable.
+                with IndexStorage(
+                    store_config,
+                    client_factory=self.chroma_clients,
+                ) as storage:
+                    dropped = storage.drop_all_collections()
+            repository.discard_generations()
+            return {**summary, "collections": dropped}

src/vidxp/infrastructure/local_snapshots.py

@@ -120,10 +120,24 @@ def read_active(self, *, required: bool = False) -> IndexSnapshot | None:
         resolved = self._read_active(required=required)
         return None if resolved is None else resolved[1]
 
+    def read_active_document(
+        self,
+    ) -> tuple[ActiveSnapshotPointer, IndexSnapshot] | None:
+        """Read the active snapshot without validating the generations.
+
+        The snapshot document is still checked for integrity and identity.
+        Only the generation manifests it points at are left unread, so a
+        repository whose generations were written by an incompatible index
+        schema can still be inspected and discarded.
+        """
+
+        return self._read_active(validate_generations=False)
+
     def _read_active(
         self,
         *,
         required: bool = False,
+        validate_generations: bool = True,
     ) -> tuple[ActiveSnapshotPointer, IndexSnapshot] | None:
         if not self.active_pointer.is_file():
             if required:
@@ -138,6 +152,7 @@ def _read_active(
             snapshot = self.read_snapshot(
                 pointer.snapshot_id,
                 expected_sha256=pointer.snapshot_sha256,
+                validate_generations=validate_generations,
             )
             return pointer, snapshot
         except IndexSchemaError:
@@ -152,6 +167,7 @@ def read_snapshot(
         snapshot_id: str,
         *,
         expected_sha256: str | None = None,
+        validate_generations: bool = True,
     ) -> IndexSnapshot:
         snapshot_path = self._snapshot_path(snapshot_id)
         if not snapshot_path.is_file():
@@ -177,7 +193,8 @@ def read_snapshot(
             raise IndexSchemaError(
                 "The snapshot filename and document identifier differ."
             )
-        self._validate_generations(snapshot)
+        if validate_generations:
+            self._validate_generations(snapshot)
         return snapshot
 
     def _validate_generations(self, snapshot: IndexSnapshot) -> None:
@@ -348,6 +365,63 @@ def clear(self) -> bool:
         )
         return True
 
+    def describe_active_generations(self) -> dict[str, Any]:
+        """Report what the active snapshot holds without validating it.
+
+        Read-only, so a caller can tell the user exactly which generated data
+        a reset would give up before anything is destroyed.
+        """
+
+        resolved = self.read_active_document()
+        if resolved is None:
+            return {
+                "discarded": False,
+                "media_ids": (),
+                "generation_ids": (),
+                "modalities": (),
+            }
+        _, snapshot = resolved
+        media_ids = tuple(sorted(snapshot.generations))
+        return {
+            "discarded": bool(media_ids),
+            "media_ids": media_ids,
+            "generation_ids": tuple(
+                snapshot.generations[media_id].generation_id
+                for media_id in media_ids
+            ),
+            "modalities": tuple(
+                sorted(
+                    {
+                        modality
+                        for reference in snapshot.generations.values()
+                        for modality in reference.modalities
+                    }
+                )
+            ),
+        }
+
+    def discard_generations(self) -> bool:
+        """Publish an empty active snapshot, whatever the generations hold.
+
+        Unlike `clear`, this does not validate the generations it replaces, so
+        it can recover a repository whose generations were written by an
+        incompatible index schema. Retained generation directories are left on
+        disk, exactly as `clear` leaves them.
+        """
+
+        resolved = self.read_active_document()
+        if resolved is None:
+            return False
+        _, snapshot = resolved
+        if not snapshot.generations:
+            return False
+        self._publish(
+            generations={},
+            config_fingerprint=snapshot.config_fingerprint,
+            configuration=dict(snapshot.configuration),
+        )
+        return True
+
     def _publish(
         self,
         *,

src/vidxp/ports.py

@@ -430,6 +430,8 @@ def remove(self, config: IndexConfig, media_id: str) -> bool: ...
 
     def clear(self, config: IndexConfig) -> bool: ...
 
+    def reset(self, config: IndexConfig) -> dict[str, Any]: ...
+
 
 class JobBackend(Protocol):
     """Durable lifecycle operations owned by the workflow engine."""