fix(webapp,clickhouse): harden bounded global log search

Author: carderne

.server-changes/improve-global-log-search.md

@@ -3,4 +3,4 @@ area: webapp
 type: improvement
 ---
 
-Global log search now uses a bounded search index, clearer time-range expansion, and partial-result feedback for expensive searches.
+Global log search now supports a bounded search index and clearer time-range expansion while keeping existing search history available during rollout.

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -840,55 +840,59 @@ export function SideMenu({
               } satisfies SideMenuItemConfig,
             ]
           : []),
-        {
-          id: "errors",
-          name: "Errors",
-          icon: BugIcon,
-          activeIconColor: "text-errors",
-          to: v3ErrorsPath(organization, project, environment),
-          dataAction: "errors",
-        },
-        {
-          id: "query",
-          name: "Query",
-          icon: CodeSquareIcon,
-          activeIconColor: "text-query",
-          to: queryPath(organization, project, environment),
-          dataAction: "query",
-        },
-        {
-          id: "queues",
-          name: "Queues",
-          icon: QueuesIcon,
-          activeIconColor: "text-queues",
-          to: v3QueuesPath(organization, project, environment),
-          dataAction: "queues",
-        },
-        {
-          id: "dashboards",
-          name: "Dashboards",
-          icon: ChartBarIcon,
-          activeIconColor: "text-metrics",
-          to: v3DashboardsLandingPath(organization, project, environment),
-          dataAction: "dashboards-landing",
-          action: (
-            <CreateDashboardButton
-              organization={organization}
-              project={project}
-              environment={environment}
-              isCollapsed={isCollapsed}
-            />
-          ),
-          after: (
-            <DashboardList
-              organization={organization}
-              project={project}
-              environment={environment}
-              isCollapsed={isCollapsed}
-              user={user}
-            />
-          ),
-        },
+        ...(isAdmin || featureFlags.hasQueryAccess
+          ? [
+              {
+                id: "errors",
+                name: "Errors",
+                icon: BugIcon,
+                activeIconColor: "text-errors",
+                to: v3ErrorsPath(organization, project, environment),
+                dataAction: "errors",
+              },
+              {
+                id: "query",
+                name: "Query",
+                icon: CodeSquareIcon,
+                activeIconColor: "text-query",
+                to: queryPath(organization, project, environment),
+                dataAction: "query",
+              },
+              {
+                id: "queues",
+                name: "Queues",
+                icon: QueuesIcon,
+                activeIconColor: "text-queues",
+                to: v3QueuesPath(organization, project, environment),
+                dataAction: "queues",
+              },
+              {
+                id: "dashboards",
+                name: "Dashboards",
+                icon: ChartBarIcon,
+                activeIconColor: "text-metrics",
+                to: v3DashboardsLandingPath(organization, project, environment),
+                dataAction: "dashboards-landing",
+                action: (
+                  <CreateDashboardButton
+                    organization={organization}
+                    project={project}
+                    environment={environment}
+                    isCollapsed={isCollapsed}
+                  />
+                ),
+                after: (
+                  <DashboardList
+                    organization={organization}
+                    project={project}
+                    environment={environment}
+                    isCollapsed={isCollapsed}
+                    user={user}
+                  />
+                ),
+              },
+            ]
+          : []),
       ],
     });
   }

apps/webapp/app/env.server.ts

@@ -1925,6 +1925,10 @@ const EnvironmentSchema = z
       .nonnegative()
       .optional(),
 
+    // v2 is populated forward-only. Keep reads on v1 until v2 has enough history or has been
+    // backfilled, then opt in explicitly per deployment.
+    LOGS_SEARCH_TABLE_VERSION: z.enum(["v1", "v2"]).default("v1"),
+
     // Logs list pagination tuning.
     LOGS_LIST_DEFAULT_PAGE_SIZE: z.coerce.number().int().positive().default(50),
     LOGS_LIST_MAX_PAGE_SIZE: z.coerce.number().int().positive().default(100),

apps/webapp/app/presenters/v3/LogsListPresenter.server.ts

@@ -15,13 +15,10 @@ import {
   convertDateToClickhouseDateTime,
 } from "~/v3/eventRepository/clickhouseEventRepository.server";
 import { ServiceValidationError } from "~/v3/services/baseService.server";
+import { escapeClickHouseLike, normalizeLogsSearchTerm } from "~/utils/logSearch";
 
 export type { LogLevel };
 
-function escapeClickHouseString(val: string): string {
-  return val.replace(/\\/g, "\\\\").replace(/\//g, "\\/").replace(/%/g, "\\%").replace(/_/g, "\\_");
-}
-
 export type LogsListOptions = {
   userId?: string;
   projectId: string;
@@ -236,15 +233,20 @@ export class LogsListPresenter extends BasePresenter {
     const now = new Date();
     const clampedTo = effectiveTo !== undefined ? (effectiveTo > now ? now : effectiveTo) : now;
 
+    const rawSearchTerm = search?.trim() ?? "";
+    const normalizedSearchTerm =
+      env.LOGS_SEARCH_TABLE_VERSION === "v2"
+        ? normalizeLogsSearchTerm(rawSearchTerm)
+        : rawSearchTerm.toLocaleLowerCase();
     const searchTerm =
-      search && search.trim() !== ""
-        ? escapeClickHouseString(search.trim()).toLowerCase()
-        : undefined;
+      normalizedSearchTerm === "" ? undefined : escapeClickHouseLike(normalizedSearchTerm);
 
     // Run exactly one bounded query. Broadening a search window is an explicit user action;
     // silently rescanning the same recent rows makes absence queries needlessly expensive.
     const runQuery = () => {
-      const queryBuilder = this.clickhouse.taskEventsSearch.logsListQueryBuilder();
+      const queryBuilder = this.clickhouse.taskEventsSearch.logsListQueryBuilder(
+        env.LOGS_SEARCH_TABLE_VERSION
+      );
 
       // The materialized view excludes events without a trace_id; this guards the legacy tail.
       queryBuilder.where("trace_id != ''");
@@ -274,13 +276,19 @@ export class LogsListPresenter extends BasePresenter {
         queryBuilder.where("run_id = {runId: String}", { runId });
       }
 
-      // search_text is normalized and indexed as one field. A single predicate is required for
-      // ClickHouse to use the text index; the old message OR attributes predicate defeated both
-      // skip indexes.
       if (searchTerm !== undefined) {
-        queryBuilder.where("search_text LIKE {searchPattern: String}", {
-          searchPattern: `%${searchTerm}%`,
-        });
+        if (env.LOGS_SEARCH_TABLE_VERSION === "v2") {
+          // One predicate lets the text index answer substring searches without an OR across
+          // independently indexed columns.
+          queryBuilder.where("search_text LIKE {searchPattern: String}", {
+            searchPattern: `%${searchTerm}%`,
+          });
+        } else {
+          queryBuilder.where(
+            "(lowerUTF8(message) LIKE {searchPattern: String} OR lowerUTF8(attributes_text) LIKE {searchPattern: String})",
+            { searchPattern: `%${searchTerm}%` }
+          );
+        }
       }
 
       if (levels && levels.length > 0) {
@@ -331,20 +339,18 @@ export class LogsListPresenter extends BasePresenter {
       // Limit + 1 to check if there are more results
       queryBuilder.limit(effectivePageSize + 1);
 
-      return queryBuilder.executeWithStats();
+      return queryBuilder.execute();
     };
 
     const [queryError, queryResult] = await runQuery();
     if (queryError) {
       throw queryError;
     }
 
-    const records = queryResult?.rows ?? [];
-    const partial =
-      Number(queryResult?.stats.read_rows ?? 0) >= env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ ||
-      Number(queryResult?.stats.elapsed_ns ?? 0) >=
-        env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME * 1_000_000_000;
-    const results = records;
+    // ClickHouse's break overflow modes can return a short prefix without a reliable completion
+    // marker. Keep the default throw behavior so the product never presents truncated results as
+    // complete.
+    const results = queryResult ?? [];
     const hasMore = results.length > effectivePageSize;
     const logs = results.slice(0, effectivePageSize);
 
@@ -413,11 +419,10 @@ export class LogsListPresenter extends BasePresenter {
       },
       hasFilters,
       hasAnyLogs: transformedLogs.length > 0,
-      partial,
       searchTerm: search,
       searchExpansion:
         searchTerm !== undefined && time.isDefault && transformedLogs.length === 0
-          ? { nextPeriod: "7d" as const }
+          ? { nextPeriod: `${Math.min(retentionLimitDays ?? 7, 7)}d` }
           : undefined,
       retention:
         retentionLimitDays !== undefined

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx

@@ -427,26 +427,21 @@ function LogsList({
 
   const expandSearch = useCallback(() => {
     const url = new URL(window.location.href);
-    url.searchParams.set("period", "7d");
+    url.searchParams.set("period", list.searchExpansion?.nextPeriod ?? "7d");
     url.searchParams.delete("cursor");
     url.searchParams.delete("log");
     navigate(`${url.pathname}?${url.searchParams.toString()}`);
-  }, [navigate]);
+  }, [list.searchExpansion?.nextPeriod, navigate]);
 
   return (
     <div className="flex min-h-0 flex-1 flex-col">
-      {list.partial && (
-        <Callout variant="warning" className="m-2 mb-0">
-          Showing partial results. Refine your search or narrow the time range.
-        </Callout>
-      )}
       {list.searchExpansion && (
         <Callout
           variant="info"
           className="m-2 mb-0"
           cta={
             <Button variant="tertiary/small" onClick={expandSearch}>
-              Search last 7 days
+              Search last {list.searchExpansion.nextPeriod.replace("d", " days")}
             </Button>
           }
         >

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx

@@ -119,7 +119,6 @@ import {
   v3RunRedirectPath,
   v3RunSpanPath,
   v3RunStreamingPath,
-  v3LogsPath,
   v3RunsPath,
 } from "~/utils/pathBuilder";
 import type { SpanOverride } from "~/v3/eventRepository/eventRepository.types";
@@ -496,12 +495,6 @@ export default function Page() {
         />
         {environment.type === "DEVELOPMENT" && <DevDisconnectedBanner isConnected={isConnected} />}
         <PageAccessories>
-          <LinkButton
-            to={`${v3LogsPath(organization, project, environment)}?runId=${encodeURIComponent(run.friendlyId)}`}
-            variant="secondary/small"
-          >
-            Search this run's logs
-          </LinkButton>
           <AdminDebugTooltip>
             <Property.Table>
               <Property.Item>

apps/webapp/app/routes/api.v1.logs.ts

@@ -87,7 +87,5 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   return json({
     logs: result.logs,
     pagination: result.pagination,
-    partial: result.partial,
-    searchExpansion: result.searchExpansion,
   });
 };

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts

@@ -0,0 +1,14 @@
+import { describe, expect, it } from "vitest";
+import { escapeClickHouseLike, normalizeLogsSearchTerm } from "./logSearch";
+
+describe("log search normalization", () => {
+  it("normalizes punctuation while preserving unicode, paths, and ids", () => {
+    expect(
+      normalizeLogsSearchTerm("TypeError: Zahlungsübersicht failed, retrying (/api/orders/42)")
+    ).toBe("typeerror: zahlungsübersicht failed retrying /api/orders/42");
+  });
+
+  it("escapes LIKE wildcards without escaping path separators", () => {
+    expect(escapeClickHouseLike("/api/a_b/100%")).toBe("/api/a\\_b/100\\%");
+  });
+});

apps/webapp/app/utils/logSearch.test.ts

@@ -0,0 +1,11 @@
+export function escapeClickHouseLike(value: string): string {
+  return value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
+}
+
+// Must match the normalization in ClickHouse migration 038.
+export function normalizeLogsSearchTerm(value: string): string {
+  return value
+    .toLocaleLowerCase()
+    .replace(/[^\p{L}\p{N}_./:@+-]+/gu, " ")
+    .trim();
+}