feat: kNN auto-router, compare tabs, per-assistant routing controls, PWA

AGENTS.md

@@ -669,6 +669,18 @@ any connector type by introspecting `credential_json` keys.
 Backend just uses the existing `PATCH /api/manage/admin/connector/{id}`
 with `disabled: bool` flipped — no special bulk endpoint needed.
 
+### Populate a local env from prod (Vespa + assistants/doc-sets)
+
+To test search / assistants / the auto-router locally against realistic
+data, `backend/scripts/clone_prod_to_local.py` copies the **latest N
+docs per source** from prod Vespa (with embeddings / ACLs / doc-set
+membership) plus the personas, prompts, and document sets from the prod
+DB into your local Vespa + Postgres. Two phases — `export` runs inside a
+prod pod, `import` runs locally — connected by a `kubectl cp`'d bundle.
+Requires the **same embedding model** locally (the script enforces it and
+aborts on mismatch). Full steps + caveats:
+[`docs/clone-prod-to-local.md`](./docs/clone-prod-to-local.md).
+
 ---
 
 ## Conventions

backend/alembic/versions/31f30b318163_persona_is_router_candidate.py

@@ -0,0 +1,35 @@
+"""persona is_router_candidate
+
+Revision ID: 31f30b318163
+Revises: a7b8c9d0e1f2
+Create Date: 2026-07-16 00:00:00.000000
+
+Adds `is_router_candidate` to persona so an admin can exclude an assistant from
+the auto-routed Search tab (both the keyword route and the kNN fallback) while
+keeping it manually selectable. NOT NULL with server_default true, so existing
+rows keep participating in routing (no behavior change on upgrade).
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "31f30b318163"
+down_revision = "a7b8c9d0e1f2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column(
+            "is_router_candidate",
+            sa.Boolean(),
+            nullable=False,
+            server_default="true",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "is_router_candidate")

backend/alembic/versions/a7b8c9d0e1f2_chat_feedback_sme_verification.py

@@ -0,0 +1,34 @@
+"""chat_feedback SME verification
+
+Revision ID: a7b8c9d0e1f2
+Revises: f3a4b5c6d7e8
+Create Date: 2026-07-07 00:00:00.000000
+
+Adds sme_verified_by / sme_verified_at to chat_feedback so the Slack "Verified by
+an SME" flow can record who verified an answer and when (extends the existing
+feedback table rather than adding a new one).
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "a7b8c9d0e1f2"
+down_revision = "f3a4b5c6d7e8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_feedback",
+        sa.Column("sme_verified_by", sa.String(), nullable=True),
+    )
+    op.add_column(
+        "chat_feedback",
+        sa.Column("sme_verified_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_feedback", "sme_verified_at")
+    op.drop_column("chat_feedback", "sme_verified_by")

backend/alembic/versions/c0d1e2f3a4b5_persona_routing_instructions.py

@@ -0,0 +1,35 @@
+"""persona: add routing_instructions (router-only metadata)
+
+Adds persona.routing_instructions — admin-editable free-text guidance read ONLY
+by the auto-routed Search tab's assistant router (which assistant to pick for a
+question). It is NEVER rendered in the user-facing UI; `description` stays the
+short human-facing label. Nullable with no backfill — the router falls back to
+`description` when this is blank, so existing assistants route on what they have
+today until an admin fills this in. See db/models.py::Persona.routing_instructions
+and secondary_llm_flows/assistant_router.
+
+Revision ID: c0d1e2f3a4b5
+Revises: b9c0d1e2f3a4
+Create Date: 2026-06-29
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "c0d1e2f3a4b5"
+down_revision = "b9c0d1e2f3a4"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("routing_instructions", sa.Text(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "routing_instructions")

backend/alembic/versions/d1e2f3a4b5c6_persona_routing_keywords.py

@@ -0,0 +1,34 @@
+"""persona: add routing_keywords (deterministic keyword pre-route)
+
+Adds persona.routing_keywords — admin-editable, comma-separated phrases that
+deterministically route a question to this assistant (case-insensitive substring
+match) BEFORE the LLM router runs. Nullable, no backfill; blank means no keyword
+override and the LLM router decides as before. Additive — existing routing is
+unchanged when no keyword matches. See db/models.py::Persona.routing_keywords and
+secondary_llm_flows/assistant_router.keyword_route.
+
+Revision ID: d1e2f3a4b5c6
+Revises: c0d1e2f3a4b5
+Create Date: 2026-06-30
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "d1e2f3a4b5c6"
+down_revision = "c0d1e2f3a4b5"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("routing_keywords", sa.Text(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "routing_keywords")

backend/alembic/versions/e2f3a4b5c6d7_persona_routing_intents.py

@@ -0,0 +1,35 @@
+"""persona: add routing_intents (semantic intent pre-route)
+
+Adds persona.routing_intents — admin-editable, NEWLINE-separated natural-language
+intent phrases (one per line). The auto-router embeds them and routes a question
+to this assistant when its nearest exemplar clears a similarity gate, BETWEEN the
+keyword pre-route and the LLM router. Nullable, no backfill; blank means no
+semantic override and routing is unchanged. Stored in Postgres only (never in
+Vespa); embeddings are computed on demand and cached per-assistant. See
+db/models.py::Persona.routing_intents and assistant_router.intent_route.
+
+Revision ID: e2f3a4b5c6d7
+Revises: d1e2f3a4b5c6
+Create Date: 2026-07-01
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "e2f3a4b5c6d7"
+down_revision = "d1e2f3a4b5c6"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("routing_intents", sa.Text(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "routing_intents")

backend/alembic/versions/f3a4b5c6d7e8_chat_referral.py

@@ -0,0 +1,55 @@
+"""chat_referral
+
+Revision ID: f3a4b5c6d7e8
+Revises: e2f3a4b5c6d7
+Create Date: 2026-07-05 00:00:00.000000
+
+Adds the chat_referral table: one row per landing on the chat UI from an external
+referral (e.g. a per-channel Slack "Ask Darwin" workflow), so inbound traffic can
+be measured by source / channel / assistant with plain SQL.
+"""
+from alembic import op
+import sqlalchemy as sa
+import fastapi_users_db_sqlalchemy
+
+# revision identifiers, used by Alembic.
+revision = "f3a4b5c6d7e8"
+down_revision = "e2f3a4b5c6d7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "chat_referral",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("utm_source", sa.String(), nullable=False),
+        sa.Column("utm_medium", sa.String(), nullable=True),
+        sa.Column("utm_campaign", sa.String(), nullable=True),
+        sa.Column("utm_channel", sa.String(), nullable=True),
+        sa.Column("assistant_name", sa.String(), nullable=True),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(["user_id"], ["user.id"]),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # Traffic queries filter/group by source and time.
+    op.create_index(
+        "ix_chat_referral_source_created",
+        "chat_referral",
+        ["utm_source", "created_at"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_chat_referral_source_created", table_name="chat_referral")
+    op.drop_table("chat_referral")