perf(query-engine): ClickStack-style skip indexes + token-aware log search

apps/api/package.json

@@ -20,6 +20,7 @@
 		"tinybird:build": "tinybird build",
 		"tinybird:deploy": "tinybird deploy",
 		"bench:fetch": "bun run scripts/bench-queries.ts fetch",
+		"bench:suite": "bun run scripts/bench-queries.ts suite",
 		"bench:run": "bun run scripts/bench-queries.ts run",
 		"bench:inspect": "bun run scripts/bench-queries.ts inspect",
 		"bench:compare": "bun run scripts/bench-queries.ts compare",

apps/api/scripts/BENCH.md

@@ -15,18 +15,65 @@ bun bench:fetch    [--context name] [--profile name] [--since 24h]
                    [--top 20] [--out path] [--org id]
                    # mine recent db.query.text spans from prod traces → JSON
 
+bun bench:suite    [--org id] [--since 24h] [--trace-id id] [--service name]
+                   [--search phrase] [--attr-key k] [--attr-value v]
+                   [--attr-scope span] [--table-map from=to,...] [--out path]
+                   # compile the committed DSL benchmark suite → JSON (same
+                   # shape as `fetch`, so `run`/`inspect`/`compare` replay it)
+
 bun bench:run      <file> [--runs 5] [--warmup 1] [--out path]
                    # replay each query N times and report aggregated stats
 
 bun bench:inspect  <file>
                    # run EXPLAIN and EXPLAIN PIPELINE for each query
 
-bun bench:compare  <a.json> <b.json>
-                   # diff two run outputs (p95 wall, read bytes, memory)
+bun bench:compare  <a.json> <b.json> [--max-regression-pct N]
+                   # diff two run outputs (p95 wall, read bytes, memory);
+                   # exits non-zero if any case's p95 regresses beyond N%
 ```
 
 Output JSONs land in `apps/api/scripts/.bench/` by default (gitignored).
 
+## Two ways to get a query set
+
+- **`fetch`** mines whatever prod actually ran for a context label — great for
+  chasing a specific slow query, but not repeatable (the fingerprints drift as
+  traffic changes).
+- **`suite`** compiles a **committed, repeatable** set of ~13 representative
+  cases (needle-in-haystack body search, trace point lookup, attribute
+  equality/contains filters, list scans, facets, autocomplete, plus canaries)
+  straight from the `@maple/query-engine` DSL, so the benchmark runs the exact
+  SQL production emits. Definitions live in
+  [scripts/bench/suite.ts](bench/suite.ts). Pass `--trace-id` / `--service` /
+  `--attr-key` / `--attr-value` that exist in the target window for
+  representative timing — the two lookup cases fall back to zero-row
+  placeholders (with a warning) otherwise.
+
+## A/B: schema-change spike (e.g. sort-key experiment)
+
+`suite --table-map` rewrites `FROM`/`JOIN` table names in the compiled SQL so
+you can benchmark the same queries against shadow tables without recompiling.
+Rules are whole-identifier and longest-first, so `traces=traces_t2` never
+touches `trace_detail_spans` or `traces_aggregates_hourly`.
+
+```
+# control vs candidate tables, same window/org
+bun bench:suite --org org_x --trace-id <id> --service <svc> \
+  --out .bench/suiteA.json
+bun bench:suite --org org_x --trace-id <id> --service <svc> \
+  --table-map "traces=traces_t2,logs=logs_l2" --out .bench/suiteB.json
+
+bun bench:run .bench/suiteA.json --out .bench/resA.json   # CLICKHOUSE_URL = dedicated CH
+bun bench:run .bench/suiteB.json --out .bench/resB.json
+bun bench:compare .bench/resA.json .bench/resB.json --max-regression-pct 25
+```
+
+Run A/B against a **dedicated ClickHouse** (docker/staging), not a Tinybird
+branch — Tinybird branches share prod compute, so wall-time is noisy (lean on
+`read rows`/`read bytes`, which are deterministic). The `inspect` command's
+`EXPLAIN PIPELINE` tells you whether `optimize_read_in_order` actually engaged
+for the candidate sort key.
+
 ## Implementation
 
 Built on Effect v4 end-to-end:

apps/api/scripts/bench-queries.ts

@@ -39,6 +39,8 @@ import {
 import { Argument, Command, Flag } from "effect/unstable/cli"
 import { BunRuntime, BunServices } from "@effect/platform-bun"
 import { CH } from "@maple/query-engine"
+import { buildSuite } from "./bench/suite"
+import { parseTableMap, remapTables } from "./bench/table-map"
 
 // ---------------------------------------------------------------------------
 // Errors
@@ -86,6 +88,13 @@ class InvalidDurationError extends Schema.TaggedErrorClass<InvalidDurationError>
 	},
 ) {}
 
+class RegressionThresholdError extends Schema.TaggedErrorClass<RegressionThresholdError>()(
+	"@maple/api/scripts/bench-queries/RegressionThresholdError",
+	{
+		message: Schema.String,
+	},
+) {}
+
 // ---------------------------------------------------------------------------
 // Internal data shapes (typed JSON; not branded — local dev tool)
 // ---------------------------------------------------------------------------
@@ -598,7 +607,7 @@ const fetchHandler = Effect.fn("bench.fetch")(function* (config: FetchConfig) {
 		() => `apps/api/scripts/.bench/queries-${timestampSlug()}.json`,
 	)
 	const output: FetchOutput = {
-		fetchedAt: now.toISOString(),
+		fetchedAt: new Date(nowMs).toISOString(),
 		source: host,
 		criteria: {
 			orgId,
@@ -638,6 +647,108 @@ const fetchHandler = Effect.fn("bench.fetch")(function* (config: FetchConfig) {
 	yield* Console.log(`\nWrote ${outputPath}`)
 })
 
+// Non-empty sentinels so the point-lookup and service-scoped cases keep their
+// query SHAPE (right table + filter) even when the user doesn't pass real
+// values. They match zero rows, so timing for those two cases is only
+// representative once real values are supplied — the handler warns when unset.
+const SUITE_PLACEHOLDER_TRACE_ID = "REPLACE_WITH_REAL_TRACE_ID"
+const SUITE_PLACEHOLDER_SERVICE = "REPLACE_WITH_REAL_SERVICE"
+
+interface SuiteConfig {
+	readonly org: Option.Option<string>
+	readonly since: string
+	readonly traceId: string
+	readonly service: string
+	readonly search: string
+	readonly attrKey: string
+	readonly attrValue: string
+	readonly attrScope: string
+	readonly tableMap: Option.Option<string>
+	readonly out: Option.Option<string>
+}
+
+// Compile the committed suite into a `fetch`-shaped JSON file so the existing
+// `run` / `inspect` / `compare` commands replay it unchanged. For the sort-key
+// A/B spike, run this twice with different `--table-map` targets and diff the
+// two `run` outputs.
+const suiteHandler = Effect.fn("bench.suite")(function* (config: SuiteConfig) {
+	const sinceMs = yield* parseRelativeDuration(config.since)
+	const nowMs = yield* Clock.currentTimeMillis
+	const startTime = formatCHDateTime(new Date(nowMs - sinceMs))
+	const endTime = formatCHDateTime(new Date(nowMs))
+	const orgId = Option.getOrElse(config.org, () => "internal")
+
+	const tableMap = yield* Effect.try({
+		try: () =>
+			parseTableMap(Option.match(config.tableMap, { onNone: () => [], onSome: (s) => s.split(",") })),
+		catch: (cause) => new MissingConfigError({ what: "--table-map", message: String(cause) }),
+	})
+
+	const cases = buildSuite({
+		traceId: config.traceId,
+		serviceName: config.service,
+		searchTerm: config.search,
+		attrKey: config.attrKey,
+		attrValue: config.attrValue,
+		attrScope: config.attrScope,
+	})
+
+	const samples: ReadonlyArray<Sample> = cases.map((c) => ({
+		fingerprint: c.name,
+		context: c.name,
+		profile: c.profile,
+		sampleSql: remapTables(c.compile({ orgId, startTime, endTime }), tableMap),
+		sampleCount: 0,
+		p50DurationMs: 0,
+		p95DurationMs: 0,
+		p99DurationMs: 0,
+		maxDurationMs: 0,
+	}))
+
+	const remapNote = Option.match(config.tableMap, {
+		onNone: () => "",
+		onSome: (s) => `  table-map: ${s}`,
+	})
+	const source = `suite (org=${orgId}, window=${config.since})${remapNote}`
+	const outputPath = Option.getOrElse(
+		config.out,
+		() => `apps/api/scripts/.bench/suite-${timestampSlug()}.json`,
+	)
+	const output: FetchOutput = {
+		fetchedAt: new Date(nowMs).toISOString(),
+		source,
+		criteria: { orgId, startTime, endTime, topN: cases.length },
+		samples,
+	}
+	yield* writeJsonFile(outputPath, output)
+
+	yield* Console.log(
+		renderTable(
+			`Benchmark suite (${cases.length} cases)`,
+			[
+				{ header: "case", width: 28 },
+				{ header: "profile", width: 12 },
+				{ header: "description", width: 56 },
+			],
+			cases.map((c) => [c.name, c.profile, c.description]),
+		),
+	)
+	yield* Console.log(`  org: ${orgId}   window: ${startTime} → ${endTime} (${config.since})${remapNote}`)
+
+	const placeholders: string[] = []
+	if (config.traceId === SUITE_PLACEHOLDER_TRACE_ID) placeholders.push("--trace-id (trace_point_lookup)")
+	if (config.service === SUITE_PLACEHOLDER_SERVICE) placeholders.push("--service (service_span_search)")
+	if (placeholders.length > 0) {
+		yield* Console.log(
+			`  ⚠ using placeholder ${placeholders.join(" and ")} — those cases match 0 rows; ` +
+				"pass real values for representative timing.",
+		)
+	}
+
+	yield* Console.log(`\nWrote ${outputPath}`)
+	yield* Console.log(`  Replay:  bun run scripts/bench-queries.ts run ${outputPath}`)
+})
+
 const stripTrailingSemi = (sql: string) => sql.replace(/;\s*$/, "")
 
 const failedResult = (sample: Sample, message: string): SampleResult => ({
@@ -871,6 +982,7 @@ const inspectHandler = Effect.fn("bench.inspect")(function* (config: { readonly
 const compareHandler = Effect.fn("bench.compare")(function* (config: {
 	readonly baseline: string
 	readonly candidate: string
+	readonly maxRegressionPct: Option.Option<number>
 }) {
 	const a = yield* readJsonFile<RunOutput>(config.baseline)
 	const b = yield* readJsonFile<RunOutput>(config.candidate)
@@ -915,6 +1027,33 @@ const compareHandler = Effect.fn("bench.compare")(function* (config: {
 			rows,
 		),
 	)
+
+	// Optional CI gate: fail if any case's candidate p95 regresses beyond the
+	// threshold vs the baseline. Cases missing from either side are ignored.
+	if (Option.isSome(config.maxRegressionPct)) {
+		const limit = config.maxRegressionPct.value
+		const regressions = a.results.flatMap((aResult) => {
+			const bResult = bByFp.get(aResult.fingerprint)
+			if (!bResult) return []
+			const base = aResult.aggregates.p95WallMs
+			const cand = bResult.aggregates.p95WallMs
+			if (!Number.isFinite(base) || !Number.isFinite(cand) || base <= 0) return []
+			const pct = ((cand - base) / base) * 100
+			return pct > limit ? [{ name: aResult.context || aResult.fingerprint, pct }] : []
+		})
+		if (regressions.length > 0) {
+			const detail = regressions
+				.map((r) => `  ${r.name}: +${r.pct.toFixed(1)}% p95 (limit +${limit}%)`)
+				.join("\n")
+			yield* Console.log(`\n${regressions.length} case(s) regressed beyond +${limit}%:\n${detail}`)
+			return yield* Effect.fail(
+				new RegressionThresholdError({
+					message: `${regressions.length} case(s) exceeded the +${limit}% p95 regression threshold`,
+				}),
+			)
+		}
+		yield* Console.log(`\nNo case regressed beyond +${limit}% p95.`)
+	}
 })
 
 // ---------------------------------------------------------------------------
@@ -968,13 +1107,62 @@ const compareCommand = Command.make(
 	{
 		baseline: Argument.string("baseline").pipe(Argument.withDescription("Baseline results JSON")),
 		candidate: Argument.string("candidate").pipe(Argument.withDescription("Candidate results JSON")),
+		maxRegressionPct: Flag.integer("max-regression-pct").pipe(
+			Flag.withDescription("Exit non-zero if any case's p95 regresses beyond this percent"),
+			Flag.optional,
+		),
 	},
 	compareHandler,
 ).pipe(Command.withDescription("Diff two run outputs (p95 wall, read bytes, memory)"))
 
+const suiteCommand = Command.make(
+	"suite",
+	{
+		org: Flag.string("org").pipe(Flag.withDescription("Org id to scope the queries (default: internal)"), Flag.optional),
+		since: Flag.string("since").pipe(
+			Flag.withDescription("Time window the queries scan, e.g. 24h or 7d"),
+			Flag.withDefault("24h"),
+		),
+		traceId: Flag.string("trace-id").pipe(
+			Flag.withDescription("Existing trace id for the point-lookup case (pass a real one for meaningful timing)"),
+			Flag.withDefault(SUITE_PLACEHOLDER_TRACE_ID),
+		),
+		service: Flag.string("service").pipe(
+			Flag.withDescription("Service name for the service-scoped canary (pass a real one for meaningful timing)"),
+			Flag.withDefault(SUITE_PLACEHOLDER_SERVICE),
+		),
+		search: Flag.string("search").pipe(
+			Flag.withDescription("Body-search phrase (an interior token engages the pre-filter, e.g. 'user login failed')"),
+			Flag.withDefault("user login failed"),
+		),
+		attrKey: Flag.string("attr-key").pipe(
+			Flag.withDescription("Attribute key for the equality/contains filter cases"),
+			Flag.withDefault("http.request.method"),
+		),
+		attrValue: Flag.string("attr-value").pipe(
+			Flag.withDescription("Attribute value for the equality/contains filter cases"),
+			Flag.withDefault("GET"),
+		),
+		attrScope: Flag.string("attr-scope").pipe(
+			Flag.withDescription("Attribute scope for the autocomplete case (span|resource|log)"),
+			Flag.withDefault("span"),
+		),
+		tableMap: Flag.string("table-map").pipe(
+			Flag.withDescription("Comma-separated FROM/JOIN rewrites for A/B, e.g. traces=traces_t2,logs=logs_l2"),
+			Flag.optional,
+		),
+		out: Flag.string("out").pipe(Flag.withDescription("Output JSON path"), Flag.optional),
+	},
+	suiteHandler,
+).pipe(
+	Command.withDescription(
+		"Compile the committed DSL benchmark suite into a fetch-shaped JSON for `run`/`compare`",
+	),
+)
+
 const rootCommand = Command.make("bench-queries").pipe(
 	Command.withDescription("Measure ClickHouse query performance"),
-	Command.withSubcommands([fetchCommand, runCommand, inspectCommand, compareCommand]),
+	Command.withSubcommands([fetchCommand, suiteCommand, runCommand, inspectCommand, compareCommand]),
 )
 
 const BenchLive = Layer.mergeAll(ClickHouse.layer, Tinybird.layer)

apps/api/scripts/bench/suite.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest"
+import { buildSuite, type SuiteInputs, type SuiteParams } from "./suite"
+
+const inputs: SuiteInputs = {
+	traceId: "trace_abc123",
+	serviceName: "maple-api",
+	searchTerm: "user login failed",
+	attrKey: "http.request.method",
+	attrValue: "GET",
+	attrScope: "span",
+}
+
+const params: SuiteParams = {
+	orgId: "org_test",
+	startTime: "2024-01-01 00:00:00",
+	endTime: "2024-01-02 00:00:00",
+}
+
+describe("buildSuite", () => {
+	const cases = buildSuite(inputs)
+
+	it("every case compiles to non-empty SQL scoped to the org", () => {
+		for (const c of cases) {
+			const sql = c.compile(params)
+			expect(sql.length, `${c.name} produced empty SQL`).toBeGreaterThan(0)
+			// Every query must be OrgId-scoped (the warehouse executor enforces this).
+			expect(sql, `${c.name} missing OrgId scope`).toContain("OrgId = 'org_test'")
+		}
+	})
+
+	it("has unique case names", () => {
+		const names = cases.map((c) => c.name)
+		expect(new Set(names).size).toBe(names.length)
+	})
+
+	it("the body-search case exercises the token index and the ILIKE tail", () => {
+		const sql = cases.find((c) => c.name === "logs_body_search")!.compile(params)
+		// "user login failed" → only the interior token "login" is index-safe.
+		expect(sql).toContain("hasToken(lower(Body), 'login')")
+		expect(sql).toContain("Body ILIKE '%user login failed%'")
+	})
+
+	it("the attribute-equality case exercises the items index", () => {
+		const sql = cases.find((c) => c.name === "traces_list_attr_equals")!.compile(params)
+		expect(sql).toContain("has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes)")
+		expect(sql).toContain("'http.request.method=GET'")
+	})
+
+	it("the point-lookup case reads the trace-detail projection", () => {
+		const sql = cases.find((c) => c.name === "trace_point_lookup")!.compile(params)
+		expect(sql).toContain("FROM trace_detail_spans")
+		expect(sql).toContain("TraceId = 'trace_abc123'")
+	})
+})