diff --git a/apps/api/package.json b/apps/api/package.json index 4c28e7ee..92ce3c99 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -20,6 +20,7 @@ "tinybird:build": "tinybird build", "tinybird:deploy": "tinybird deploy", "bench:fetch": "bun run scripts/bench-queries.ts fetch", + "bench:suite": "bun run scripts/bench-queries.ts suite", "bench:run": "bun run scripts/bench-queries.ts run", "bench:inspect": "bun run scripts/bench-queries.ts inspect", "bench:compare": "bun run scripts/bench-queries.ts compare", diff --git a/apps/api/scripts/BENCH.md b/apps/api/scripts/BENCH.md index cae7d8a5..d1601b1e 100644 --- a/apps/api/scripts/BENCH.md +++ b/apps/api/scripts/BENCH.md @@ -15,18 +15,65 @@ bun bench:fetch [--context name] [--profile name] [--since 24h] [--top 20] [--out path] [--org id] # mine recent db.query.text spans from prod traces → JSON +bun bench:suite [--org id] [--since 24h] [--trace-id id] [--service name] + [--search phrase] [--attr-key k] [--attr-value v] + [--attr-scope span] [--table-map from=to,...] [--out path] + # compile the committed DSL benchmark suite → JSON (same + # shape as `fetch`, so `run`/`inspect`/`compare` replay it) + bun bench:run [--runs 5] [--warmup 1] [--out path] # replay each query N times and report aggregated stats bun bench:inspect # run EXPLAIN and EXPLAIN PIPELINE for each query -bun bench:compare - # diff two run outputs (p95 wall, read bytes, memory) +bun bench:compare [--max-regression-pct N] + # diff two run outputs (p95 wall, read bytes, memory); + # exits non-zero if any case's p95 regresses beyond N% ``` Output JSONs land in `apps/api/scripts/.bench/` by default (gitignored). +## Two ways to get a query set + +- **`fetch`** mines whatever prod actually ran for a context label — great for + chasing a specific slow query, but not repeatable (the fingerprints drift as + traffic changes). +- **`suite`** compiles a **committed, repeatable** set of ~13 representative + cases (needle-in-haystack body search, trace point lookup, attribute + equality/contains filters, list scans, facets, autocomplete, plus canaries) + straight from the `@maple/query-engine` DSL, so the benchmark runs the exact + SQL production emits. Definitions live in + [scripts/bench/suite.ts](bench/suite.ts). Pass `--trace-id` / `--service` / + `--attr-key` / `--attr-value` that exist in the target window for + representative timing — the two lookup cases fall back to zero-row + placeholders (with a warning) otherwise. + +## A/B: schema-change spike (e.g. sort-key experiment) + +`suite --table-map` rewrites `FROM`/`JOIN` table names in the compiled SQL so +you can benchmark the same queries against shadow tables without recompiling. +Rules are whole-identifier and longest-first, so `traces=traces_t2` never +touches `trace_detail_spans` or `traces_aggregates_hourly`. + +``` +# control vs candidate tables, same window/org +bun bench:suite --org org_x --trace-id --service \ + --out .bench/suiteA.json +bun bench:suite --org org_x --trace-id --service \ + --table-map "traces=traces_t2,logs=logs_l2" --out .bench/suiteB.json + +bun bench:run .bench/suiteA.json --out .bench/resA.json # CLICKHOUSE_URL = dedicated CH +bun bench:run .bench/suiteB.json --out .bench/resB.json +bun bench:compare .bench/resA.json .bench/resB.json --max-regression-pct 25 +``` + +Run A/B against a **dedicated ClickHouse** (docker/staging), not a Tinybird +branch — Tinybird branches share prod compute, so wall-time is noisy (lean on +`read rows`/`read bytes`, which are deterministic). The `inspect` command's +`EXPLAIN PIPELINE` tells you whether `optimize_read_in_order` actually engaged +for the candidate sort key. + ## Implementation Built on Effect v4 end-to-end: diff --git a/apps/api/scripts/bench-queries.ts b/apps/api/scripts/bench-queries.ts index 76ac76eb..180d459a 100644 --- a/apps/api/scripts/bench-queries.ts +++ b/apps/api/scripts/bench-queries.ts @@ -39,6 +39,8 @@ import { import { Argument, Command, Flag } from "effect/unstable/cli" import { BunRuntime, BunServices } from "@effect/platform-bun" import { CH } from "@maple/query-engine" +import { buildSuite } from "./bench/suite" +import { parseTableMap, remapTables } from "./bench/table-map" // --------------------------------------------------------------------------- // Errors @@ -86,6 +88,13 @@ class InvalidDurationError extends Schema.TaggedErrorClass }, ) {} +class RegressionThresholdError extends Schema.TaggedErrorClass()( + "@maple/api/scripts/bench-queries/RegressionThresholdError", + { + message: Schema.String, + }, +) {} + // --------------------------------------------------------------------------- // Internal data shapes (typed JSON; not branded — local dev tool) // --------------------------------------------------------------------------- @@ -598,7 +607,7 @@ const fetchHandler = Effect.fn("bench.fetch")(function* (config: FetchConfig) { () => `apps/api/scripts/.bench/queries-${timestampSlug()}.json`, ) const output: FetchOutput = { - fetchedAt: now.toISOString(), + fetchedAt: new Date(nowMs).toISOString(), source: host, criteria: { orgId, @@ -638,6 +647,108 @@ const fetchHandler = Effect.fn("bench.fetch")(function* (config: FetchConfig) { yield* Console.log(`\nWrote ${outputPath}`) }) +// Non-empty sentinels so the point-lookup and service-scoped cases keep their +// query SHAPE (right table + filter) even when the user doesn't pass real +// values. They match zero rows, so timing for those two cases is only +// representative once real values are supplied — the handler warns when unset. +const SUITE_PLACEHOLDER_TRACE_ID = "REPLACE_WITH_REAL_TRACE_ID" +const SUITE_PLACEHOLDER_SERVICE = "REPLACE_WITH_REAL_SERVICE" + +interface SuiteConfig { + readonly org: Option.Option + readonly since: string + readonly traceId: string + readonly service: string + readonly search: string + readonly attrKey: string + readonly attrValue: string + readonly attrScope: string + readonly tableMap: Option.Option + readonly out: Option.Option +} + +// Compile the committed suite into a `fetch`-shaped JSON file so the existing +// `run` / `inspect` / `compare` commands replay it unchanged. For the sort-key +// A/B spike, run this twice with different `--table-map` targets and diff the +// two `run` outputs. +const suiteHandler = Effect.fn("bench.suite")(function* (config: SuiteConfig) { + const sinceMs = yield* parseRelativeDuration(config.since) + const nowMs = yield* Clock.currentTimeMillis + const startTime = formatCHDateTime(new Date(nowMs - sinceMs)) + const endTime = formatCHDateTime(new Date(nowMs)) + const orgId = Option.getOrElse(config.org, () => "internal") + + const tableMap = yield* Effect.try({ + try: () => + parseTableMap(Option.match(config.tableMap, { onNone: () => [], onSome: (s) => s.split(",") })), + catch: (cause) => new MissingConfigError({ what: "--table-map", message: String(cause) }), + }) + + const cases = buildSuite({ + traceId: config.traceId, + serviceName: config.service, + searchTerm: config.search, + attrKey: config.attrKey, + attrValue: config.attrValue, + attrScope: config.attrScope, + }) + + const samples: ReadonlyArray = cases.map((c) => ({ + fingerprint: c.name, + context: c.name, + profile: c.profile, + sampleSql: remapTables(c.compile({ orgId, startTime, endTime }), tableMap), + sampleCount: 0, + p50DurationMs: 0, + p95DurationMs: 0, + p99DurationMs: 0, + maxDurationMs: 0, + })) + + const remapNote = Option.match(config.tableMap, { + onNone: () => "", + onSome: (s) => ` table-map: ${s}`, + }) + const source = `suite (org=${orgId}, window=${config.since})${remapNote}` + const outputPath = Option.getOrElse( + config.out, + () => `apps/api/scripts/.bench/suite-${timestampSlug()}.json`, + ) + const output: FetchOutput = { + fetchedAt: new Date(nowMs).toISOString(), + source, + criteria: { orgId, startTime, endTime, topN: cases.length }, + samples, + } + yield* writeJsonFile(outputPath, output) + + yield* Console.log( + renderTable( + `Benchmark suite (${cases.length} cases)`, + [ + { header: "case", width: 28 }, + { header: "profile", width: 12 }, + { header: "description", width: 56 }, + ], + cases.map((c) => [c.name, c.profile, c.description]), + ), + ) + yield* Console.log(` org: ${orgId} window: ${startTime} → ${endTime} (${config.since})${remapNote}`) + + const placeholders: string[] = [] + if (config.traceId === SUITE_PLACEHOLDER_TRACE_ID) placeholders.push("--trace-id (trace_point_lookup)") + if (config.service === SUITE_PLACEHOLDER_SERVICE) placeholders.push("--service (service_span_search)") + if (placeholders.length > 0) { + yield* Console.log( + ` ⚠ using placeholder ${placeholders.join(" and ")} — those cases match 0 rows; ` + + "pass real values for representative timing.", + ) + } + + yield* Console.log(`\nWrote ${outputPath}`) + yield* Console.log(` Replay: bun run scripts/bench-queries.ts run ${outputPath}`) +}) + const stripTrailingSemi = (sql: string) => sql.replace(/;\s*$/, "") const failedResult = (sample: Sample, message: string): SampleResult => ({ @@ -871,6 +982,7 @@ const inspectHandler = Effect.fn("bench.inspect")(function* (config: { readonly const compareHandler = Effect.fn("bench.compare")(function* (config: { readonly baseline: string readonly candidate: string + readonly maxRegressionPct: Option.Option }) { const a = yield* readJsonFile(config.baseline) const b = yield* readJsonFile(config.candidate) @@ -915,6 +1027,33 @@ const compareHandler = Effect.fn("bench.compare")(function* (config: { rows, ), ) + + // Optional CI gate: fail if any case's candidate p95 regresses beyond the + // threshold vs the baseline. Cases missing from either side are ignored. + if (Option.isSome(config.maxRegressionPct)) { + const limit = config.maxRegressionPct.value + const regressions = a.results.flatMap((aResult) => { + const bResult = bByFp.get(aResult.fingerprint) + if (!bResult) return [] + const base = aResult.aggregates.p95WallMs + const cand = bResult.aggregates.p95WallMs + if (!Number.isFinite(base) || !Number.isFinite(cand) || base <= 0) return [] + const pct = ((cand - base) / base) * 100 + return pct > limit ? [{ name: aResult.context || aResult.fingerprint, pct }] : [] + }) + if (regressions.length > 0) { + const detail = regressions + .map((r) => ` ${r.name}: +${r.pct.toFixed(1)}% p95 (limit +${limit}%)`) + .join("\n") + yield* Console.log(`\n${regressions.length} case(s) regressed beyond +${limit}%:\n${detail}`) + return yield* Effect.fail( + new RegressionThresholdError({ + message: `${regressions.length} case(s) exceeded the +${limit}% p95 regression threshold`, + }), + ) + } + yield* Console.log(`\nNo case regressed beyond +${limit}% p95.`) + } }) // --------------------------------------------------------------------------- @@ -968,13 +1107,62 @@ const compareCommand = Command.make( { baseline: Argument.string("baseline").pipe(Argument.withDescription("Baseline results JSON")), candidate: Argument.string("candidate").pipe(Argument.withDescription("Candidate results JSON")), + maxRegressionPct: Flag.integer("max-regression-pct").pipe( + Flag.withDescription("Exit non-zero if any case's p95 regresses beyond this percent"), + Flag.optional, + ), }, compareHandler, ).pipe(Command.withDescription("Diff two run outputs (p95 wall, read bytes, memory)")) +const suiteCommand = Command.make( + "suite", + { + org: Flag.string("org").pipe(Flag.withDescription("Org id to scope the queries (default: internal)"), Flag.optional), + since: Flag.string("since").pipe( + Flag.withDescription("Time window the queries scan, e.g. 24h or 7d"), + Flag.withDefault("24h"), + ), + traceId: Flag.string("trace-id").pipe( + Flag.withDescription("Existing trace id for the point-lookup case (pass a real one for meaningful timing)"), + Flag.withDefault(SUITE_PLACEHOLDER_TRACE_ID), + ), + service: Flag.string("service").pipe( + Flag.withDescription("Service name for the service-scoped canary (pass a real one for meaningful timing)"), + Flag.withDefault(SUITE_PLACEHOLDER_SERVICE), + ), + search: Flag.string("search").pipe( + Flag.withDescription("Body-search phrase (an interior token engages the pre-filter, e.g. 'user login failed')"), + Flag.withDefault("user login failed"), + ), + attrKey: Flag.string("attr-key").pipe( + Flag.withDescription("Attribute key for the equality/contains filter cases"), + Flag.withDefault("http.request.method"), + ), + attrValue: Flag.string("attr-value").pipe( + Flag.withDescription("Attribute value for the equality/contains filter cases"), + Flag.withDefault("GET"), + ), + attrScope: Flag.string("attr-scope").pipe( + Flag.withDescription("Attribute scope for the autocomplete case (span|resource|log)"), + Flag.withDefault("span"), + ), + tableMap: Flag.string("table-map").pipe( + Flag.withDescription("Comma-separated FROM/JOIN rewrites for A/B, e.g. traces=traces_t2,logs=logs_l2"), + Flag.optional, + ), + out: Flag.string("out").pipe(Flag.withDescription("Output JSON path"), Flag.optional), + }, + suiteHandler, +).pipe( + Command.withDescription( + "Compile the committed DSL benchmark suite into a fetch-shaped JSON for `run`/`compare`", + ), +) + const rootCommand = Command.make("bench-queries").pipe( Command.withDescription("Measure ClickHouse query performance"), - Command.withSubcommands([fetchCommand, runCommand, inspectCommand, compareCommand]), + Command.withSubcommands([fetchCommand, suiteCommand, runCommand, inspectCommand, compareCommand]), ) const BenchLive = Layer.mergeAll(ClickHouse.layer, Tinybird.layer) diff --git a/apps/api/scripts/bench/suite.test.ts b/apps/api/scripts/bench/suite.test.ts new file mode 100644 index 00000000..be097390 --- /dev/null +++ b/apps/api/scripts/bench/suite.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest" +import { buildSuite, type SuiteInputs, type SuiteParams } from "./suite" + +const inputs: SuiteInputs = { + traceId: "trace_abc123", + serviceName: "maple-api", + searchTerm: "user login failed", + attrKey: "http.request.method", + attrValue: "GET", + attrScope: "span", +} + +const params: SuiteParams = { + orgId: "org_test", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", +} + +describe("buildSuite", () => { + const cases = buildSuite(inputs) + + it("every case compiles to non-empty SQL scoped to the org", () => { + for (const c of cases) { + const sql = c.compile(params) + expect(sql.length, `${c.name} produced empty SQL`).toBeGreaterThan(0) + // Every query must be OrgId-scoped (the warehouse executor enforces this). + expect(sql, `${c.name} missing OrgId scope`).toContain("OrgId = 'org_test'") + } + }) + + it("has unique case names", () => { + const names = cases.map((c) => c.name) + expect(new Set(names).size).toBe(names.length) + }) + + it("the body-search case exercises the token index and the ILIKE tail", () => { + const sql = cases.find((c) => c.name === "logs_body_search")!.compile(params) + // "user login failed" → only the interior token "login" is index-safe. + expect(sql).toContain("hasToken(lower(Body), 'login')") + expect(sql).toContain("Body ILIKE '%user login failed%'") + }) + + it("the attribute-equality case exercises the items index", () => { + const sql = cases.find((c) => c.name === "traces_list_attr_equals")!.compile(params) + expect(sql).toContain("has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes)") + expect(sql).toContain("'http.request.method=GET'") + }) + + it("the point-lookup case reads the trace-detail projection", () => { + const sql = cases.find((c) => c.name === "trace_point_lookup")!.compile(params) + expect(sql).toContain("FROM trace_detail_spans") + expect(sql).toContain("TraceId = 'trace_abc123'") + }) +}) diff --git a/apps/api/scripts/bench/suite.ts b/apps/api/scripts/bench/suite.ts new file mode 100644 index 00000000..3007d7be --- /dev/null +++ b/apps/api/scripts/bench/suite.ts @@ -0,0 +1,153 @@ +// --------------------------------------------------------------------------- +// bench/suite.ts — committed, repeatable query benchmark suite +// +// Each case compiles a query from the SAME `@maple/query-engine` DSL that +// production runs, so the benchmark measures the exact SQL the app emits (not a +// hand-written approximation). Cases mirror ClickStack's representative +// patterns: needle-in-haystack body search, trace point lookup, attribute +// equality/contains filters, top-N list scans, facets, and autocomplete, plus a +// couple of service-scoped canaries to catch regressions from schema changes. +// +// The suite compiles to plain SQL strings, which the `bench` CLI writes into the +// same JSON shape `fetch` produces — so `run` / `inspect` / `compare` replay it +// unchanged. Keep this pure (no IO) so it stays unit-testable. +// --------------------------------------------------------------------------- + +import { CH } from "@maple/query-engine" + +/** Params substituted at compile time (the `param.*` placeholders every query shares). */ +export interface SuiteParams { + readonly orgId: string + readonly startTime: string + readonly endTime: string +} + +/** Literal inputs baked into query construction (a real trace id, a real attribute, …). */ +export interface SuiteInputs { + /** Existing trace id for the point-lookup case (bloom + trace_detail_spans path). */ + readonly traceId: string + /** A service present in the window, for the service-scoped canary. */ + readonly serviceName: string + /** Multi-word phrase so the body-search token pre-filter actually engages. */ + readonly searchTerm: string + /** Attribute key/value that occurs in the window (drives the items-index case). */ + readonly attrKey: string + readonly attrValue: string + /** Attribute scope for the autocomplete case ("span" | "resource" | "log"). */ + readonly attrScope: string +} + +export interface SuiteCase { + readonly name: string + readonly profile: string + readonly description: string + /** Compile this case's SQL for the given window/org. */ + readonly compile: (params: SuiteParams) => string +} + +/** + * Build the benchmark suite. `inputs` supplies the literal values (trace id, + * attribute, search phrase) that must exist in the target window for the query + * to touch real data — the CLI resolves sensible defaults or takes them from + * flags. + */ +export function buildSuite(inputs: SuiteInputs): ReadonlyArray { + const q = (query: Parameters[0], p: SuiteParams): string => CH.compile(query, p).sql + const u = (union: Parameters[0], p: SuiteParams): string => + CH.compileUnion(union, p).sql + + return [ + { + name: "logs_body_search", + profile: "list", + description: "Needle-in-haystack body search — exercises idx_body_tokens + ILIKE tail", + compile: (p) => q(CH.logsListQuery({ search: inputs.searchTerm, limit: 50 }), p), + }, + { + name: "logs_count_search", + profile: "list", + description: "count(*) over a body search — pure granule-pruning win from the token index", + compile: (p) => q(CH.logsCountQuery({ search: inputs.searchTerm }), p), + }, + { + name: "trace_point_lookup", + profile: "list", + description: "All spans of one trace — trace_detail_spans (OrgId,TraceId,SpanId) sort key", + compile: (p) => q(CH.spanSearchQuery({ traceId: inputs.traceId, limit: 100 }), p), + }, + { + name: "traces_list", + profile: "list", + description: "Traces list ORDER BY Timestamp DESC LIMIT N (two-stage cutoff)", + compile: (p) => q(CH.tracesListQuery({ limit: 50 }), p), + }, + { + name: "traces_list_attr_equals", + profile: "list", + description: "Traces list filtered by attribute equality — exercises idx_span_attr_items", + compile: (p) => + q( + CH.tracesListQuery({ + attributeFilters: [{ key: inputs.attrKey, value: inputs.attrValue, mode: "equals" }], + limit: 50, + }), + p, + ), + }, + { + name: "traces_list_attr_contains", + profile: "list", + description: "Traces list filtered by attribute substring — no items index (positionCaseInsensitive)", + compile: (p) => + q( + CH.tracesListQuery({ + attributeFilters: [{ key: inputs.attrKey, value: inputs.attrValue, mode: "contains" }], + limit: 50, + }), + p, + ), + }, + { + name: "logs_list", + profile: "list", + description: "Logs list ORDER BY Timestamp DESC LIMIT N (two-stage cutoff)", + compile: (p) => q(CH.logsListQuery({ limit: 50 }), p), + }, + { + name: "traces_facets", + profile: "unbounded", + description: "Trace facet breakdown (multi-branch UNION ALL)", + compile: (p) => u(CH.tracesFacetsQuery({}), p), + }, + { + name: "services_facets", + profile: "unbounded", + description: "Service facet breakdown (4-way UNION ALL over service_overview_spans)", + compile: (p) => u(CH.servicesFacetsQuery(), p), + }, + { + name: "logs_facets", + profile: "unbounded", + description: "Log facet breakdown (UNION over logs_aggregates_hourly)", + compile: (p) => u(CH.logsFacetsQuery({}), p), + }, + { + name: "attribute_keys_autocomplete", + profile: "discovery", + description: "Attribute-key autocomplete over the attribute_keys_hourly MV", + compile: (p) => q(CH.attributeKeysQuery({ scope: inputs.attrScope, limit: 200 }), p), + }, + { + name: "top_operations", + profile: "aggregation", + description: "Top operations by count — aggregation canary (should stay flat)", + compile: (p) => q(CH.topOperationsQuery({ metric: "count", limit: 20 }), p), + }, + { + name: "service_span_search", + profile: "list", + description: "Service-scoped raw span scan — sort-key-locality canary for the PK spike", + compile: (p) => q(CH.spanSearchQuery({ serviceName: inputs.serviceName, limit: 50 }), p), + }, + ] +} diff --git a/apps/api/scripts/bench/table-map.test.ts b/apps/api/scripts/bench/table-map.test.ts new file mode 100644 index 00000000..19387088 --- /dev/null +++ b/apps/api/scripts/bench/table-map.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest" +import { parseTableMap, remapTables } from "./table-map" + +describe("parseTableMap", () => { + it("parses from=to entries and skips blanks", () => { + const map = parseTableMap(["traces=traces_t2", " logs = logs_l2 ", ""]) + expect(map.get("traces")).toBe("traces_t2") + expect(map.get("logs")).toBe("logs_l2") + expect(map.size).toBe(2) + }) + + it("rejects malformed entries", () => { + expect(() => parseTableMap(["traces"])).toThrow() + expect(() => parseTableMap(["=traces_t2"])).toThrow() + expect(() => parseTableMap(["traces="])).toThrow() + }) +}) + +describe("remapTables", () => { + const map = parseTableMap(["traces=traces_t2", "logs=logs_l2"]) + + it("rewrites FROM and JOIN table positions", () => { + expect(remapTables("SELECT * FROM traces WHERE OrgId = 'x'", map)).toBe( + "SELECT * FROM traces_t2 WHERE OrgId = 'x'", + ) + expect(remapTables("SELECT * FROM a JOIN logs ON a.id = logs.id", map)).toContain("JOIN logs_l2 ON") + }) + + it("does NOT touch longer table names that share a prefix", () => { + // `traces` rule must not clobber trace_detail_spans / traces_aggregates_hourly. + expect(remapTables("SELECT * FROM trace_detail_spans", map)).toBe("SELECT * FROM trace_detail_spans") + expect(remapTables("SELECT * FROM traces_aggregates_hourly", map)).toBe( + "SELECT * FROM traces_aggregates_hourly", + ) + }) + + it("does NOT rewrite the name outside a FROM/JOIN position", () => { + // A column or literal that happens to equal a table name is left alone. + expect(remapTables("SELECT traces FROM x WHERE note = 'traces'", map)).toBe( + "SELECT traces FROM x WHERE note = 'traces'", + ) + }) + + it("is a no-op for an empty map", () => { + const sql = "SELECT * FROM traces" + expect(remapTables(sql, parseTableMap([]))).toBe(sql) + }) +}) diff --git a/apps/api/scripts/bench/table-map.ts b/apps/api/scripts/bench/table-map.ts new file mode 100644 index 00000000..3561d330 --- /dev/null +++ b/apps/api/scripts/bench/table-map.ts @@ -0,0 +1,50 @@ +// --------------------------------------------------------------------------- +// bench/table-map.ts — rewrite table identifiers in compiled SQL for A/B runs +// +// The sort-key spike compares the same suite against shadow tables (e.g. +// `traces` vs `traces_t2`). Rather than recompile the DSL against alternate +// table names, we rewrite `FROM ` / `JOIN ` in the already-compiled +// SQL. Rules apply longest-source-name-first and are anchored on whole +// identifiers so a `traces` rule never touches `trace_detail_spans` or +// `traces_aggregates_hourly`. +// --------------------------------------------------------------------------- + +export type TableMap = ReadonlyMap + +const escapeRegExp = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + +/** + * Parse `["traces=traces_t2", "logs=logs_l2"]` (or a single comma-joined string + * split by the caller) into a source→target map. Throws on a malformed entry. + */ +export function parseTableMap(entries: ReadonlyArray): TableMap { + const map = new Map() + for (const raw of entries) { + const entry = raw.trim() + if (entry === "") continue + const eq = entry.indexOf("=") + if (eq <= 0 || eq === entry.length - 1) { + throw new Error(`Invalid --table-map entry "${raw}" (expected from=to, e.g. traces=traces_t2)`) + } + map.set(entry.slice(0, eq).trim(), entry.slice(eq + 1).trim()) + } + return map +} + +/** + * Rewrite table identifiers in `sql`. Only rewrites the table position (directly + * after `FROM` / `JOIN`), matching the whole identifier, so string literals or + * column names that happen to share a table's name are untouched. Rules are + * applied longest-source-first to avoid a short name shadowing a longer one. + */ +export function remapTables(sql: string, map: TableMap): string { + if (map.size === 0) return sql + const names = [...map.keys()].sort((a, b) => b.length - a.length) + let out = sql + for (const name of names) { + const target = map.get(name)! + const re = new RegExp(`(\\b(?:FROM|JOIN)\\s+)${escapeRegExp(name)}\\b`, "g") + out = out.replace(re, `$1${target}`) + } + return out +} diff --git a/apps/api/vitest.config.ts b/apps/api/vitest.config.ts index 81ef9aae..a2b2b316 100644 --- a/apps/api/vitest.config.ts +++ b/apps/api/vitest.config.ts @@ -14,7 +14,7 @@ export default defineConfig({ }, test: { environment: "node", - include: ["src/**/*.test.ts"], + include: ["src/**/*.test.ts", "scripts/**/*.test.ts"], // Generous timeouts: the DB-backed suites boot a fresh PGlite (WASM) per // test and some retry tests run real exponential backoff. Under CI's // parallel `turbo test`, CPU starvation stretches these past the 5s diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 8b551672..cba5117a 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,344 +1,344 @@ { - "projectRevision": "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd", - "orgPlaceholder": "__ORG__", - "datasources": { - "traces": { - "table": "traces", - "columns": [ - "OrgId", - "Timestamp", - "TraceId", - "SpanId", - "ParentSpanId", - "TraceState", - "SpanName", - "SpanKind", - "ServiceName", - "ResourceSchemaUrl", - "ResourceAttributes", - "ScopeSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "Duration", - "StatusCode", - "StatusMessage", - "SpanAttributes", - "EventsTimestamp", - "EventsName", - "EventsAttributes", - "LinksTraceId", - "LinksSpanId", - "LinksTraceState", - "LinksAttributes" - ], - "selects": [ - "__ORG__", - "start_time", - "trace_id", - "span_id", - "parent_span_id", - "trace_state", - "span_name", - "span_kind", - "service_name", - "resource_schema_url", - "resource_attributes", - "scope_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "duration", - "status_code", - "status_message", - "span_attributes", - "events_timestamp", - "events_name", - "events_attributes", - "links_trace_id", - "links_span_id", - "links_trace_state", - "links_attributes" - ], - "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))" - }, - "logs": { - "table": "logs", - "columns": [ - "OrgId", - "Timestamp", - "TimestampTime", - "TraceId", - "SpanId", - "TraceFlags", - "SeverityText", - "SeverityNumber", - "ServiceName", - "Body", - "ResourceSchemaUrl", - "ResourceAttributes", - "ScopeSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "LogAttributes" - ], - "selects": [ - "__ORG__", - "timestamp", - "timestamp", - "trace_id", - "span_id", - "flags", - "severity_text", - "severity_number", - "service_name", - "body", - "resource_schema_url", - "resource_attributes", - "scope_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "log_attributes" - ], - "inputSchema": "timestamp DateTime64(9), trace_id String, span_id String, flags UInt8, severity_text LowCardinality(String), severity_number UInt8, service_name LowCardinality(String), body String, resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), log_attributes Map(LowCardinality(String), String)" - }, - "metrics_sum": { - "table": "metrics_sum", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Value", - "Flags", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes", - "AggregationTemporality", - "IsMonotonic" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "value", - "flags", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes", - "aggregation_temporality", - "is_monotonic" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), aggregation_temporality Int32, is_monotonic Bool" - }, - "metrics_gauge": { - "table": "metrics_gauge", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Value", - "Flags", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "value", - "flags", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String))" - }, - "metrics_histogram": { - "table": "metrics_histogram", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Count", - "Sum", - "BucketCounts", - "ExplicitBounds", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes", - "Flags", - "Min", - "Max", - "AggregationTemporality" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "count", - "sum", - "bucket_counts", - "explicit_bounds", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes", - "flags", - "min", - "max", - "aggregation_temporality" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, bucket_counts Array(UInt64), explicit_bounds Array(Float64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" - }, - "metrics_exponential_histogram": { - "table": "metrics_exponential_histogram", - "columns": [ - "OrgId", - "ResourceAttributes", - "ResourceSchemaUrl", - "ScopeName", - "ScopeVersion", - "ScopeAttributes", - "ScopeSchemaUrl", - "ServiceName", - "MetricName", - "MetricDescription", - "MetricUnit", - "Attributes", - "StartTimeUnix", - "TimeUnix", - "Count", - "Sum", - "Scale", - "ZeroCount", - "PositiveOffset", - "PositiveBucketCounts", - "NegativeOffset", - "NegativeBucketCounts", - "ExemplarsTraceId", - "ExemplarsSpanId", - "ExemplarsTimestamp", - "ExemplarsValue", - "ExemplarsFilteredAttributes", - "Flags", - "Min", - "Max", - "AggregationTemporality" - ], - "selects": [ - "__ORG__", - "resource_attributes", - "resource_schema_url", - "scope_name", - "scope_version", - "scope_attributes", - "scope_schema_url", - "service_name", - "metric_name", - "metric_description", - "metric_unit", - "metric_attributes", - "start_timestamp", - "timestamp", - "count", - "sum", - "scale", - "zero_count", - "positive_offset", - "positive_bucket_counts", - "negative_offset", - "negative_bucket_counts", - "exemplars_trace_id", - "exemplars_span_id", - "exemplars_timestamp", - "exemplars_value", - "exemplars_filtered_attributes", - "flags", - "min", - "max", - "aggregation_temporality" - ], - "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, scale Int32, zero_count UInt64, positive_offset Int32, positive_bucket_counts Array(UInt64), negative_offset Int32, negative_bucket_counts Array(UInt64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" - } - } + "projectRevision": "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b", + "orgPlaceholder": "__ORG__", + "datasources": { + "traces": { + "table": "traces", + "columns": [ + "OrgId", + "Timestamp", + "TraceId", + "SpanId", + "ParentSpanId", + "TraceState", + "SpanName", + "SpanKind", + "ServiceName", + "ResourceSchemaUrl", + "ResourceAttributes", + "ScopeSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "Duration", + "StatusCode", + "StatusMessage", + "SpanAttributes", + "EventsTimestamp", + "EventsName", + "EventsAttributes", + "LinksTraceId", + "LinksSpanId", + "LinksTraceState", + "LinksAttributes" + ], + "selects": [ + "__ORG__", + "start_time", + "trace_id", + "span_id", + "parent_span_id", + "trace_state", + "span_name", + "span_kind", + "service_name", + "resource_schema_url", + "resource_attributes", + "scope_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "duration", + "status_code", + "status_message", + "span_attributes", + "events_timestamp", + "events_name", + "events_attributes", + "links_trace_id", + "links_span_id", + "links_trace_state", + "links_attributes" + ], + "inputSchema": "start_time DateTime64(9), trace_id String, span_id String, parent_span_id String, trace_state String, span_name LowCardinality(String), span_kind LowCardinality(String), service_name LowCardinality(String), resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), duration UInt64, status_code LowCardinality(String), status_message String, span_attributes Map(LowCardinality(String), String), events_timestamp Array(DateTime64(9)), events_name Array(LowCardinality(String)), events_attributes Array(Map(LowCardinality(String), String)), links_trace_id Array(String), links_span_id Array(String), links_trace_state Array(String), links_attributes Array(Map(LowCardinality(String), String))" + }, + "logs": { + "table": "logs", + "columns": [ + "OrgId", + "Timestamp", + "TimestampTime", + "TraceId", + "SpanId", + "TraceFlags", + "SeverityText", + "SeverityNumber", + "ServiceName", + "Body", + "ResourceSchemaUrl", + "ResourceAttributes", + "ScopeSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "LogAttributes" + ], + "selects": [ + "__ORG__", + "timestamp", + "timestamp", + "trace_id", + "span_id", + "flags", + "severity_text", + "severity_number", + "service_name", + "body", + "resource_schema_url", + "resource_attributes", + "scope_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "log_attributes" + ], + "inputSchema": "timestamp DateTime64(9), trace_id String, span_id String, flags UInt8, severity_text LowCardinality(String), severity_number UInt8, service_name LowCardinality(String), body String, resource_schema_url String, resource_attributes Map(LowCardinality(String), String), scope_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), log_attributes Map(LowCardinality(String), String)" + }, + "metrics_sum": { + "table": "metrics_sum", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Value", + "Flags", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes", + "AggregationTemporality", + "IsMonotonic" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "value", + "flags", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes", + "aggregation_temporality", + "is_monotonic" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), aggregation_temporality Int32, is_monotonic Bool" + }, + "metrics_gauge": { + "table": "metrics_gauge", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Value", + "Flags", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "value", + "flags", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), value Float64, flags UInt32, exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String))" + }, + "metrics_histogram": { + "table": "metrics_histogram", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Count", + "Sum", + "BucketCounts", + "ExplicitBounds", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes", + "Flags", + "Min", + "Max", + "AggregationTemporality" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "count", + "sum", + "bucket_counts", + "explicit_bounds", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes", + "flags", + "min", + "max", + "aggregation_temporality" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, bucket_counts Array(UInt64), explicit_bounds Array(Float64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" + }, + "metrics_exponential_histogram": { + "table": "metrics_exponential_histogram", + "columns": [ + "OrgId", + "ResourceAttributes", + "ResourceSchemaUrl", + "ScopeName", + "ScopeVersion", + "ScopeAttributes", + "ScopeSchemaUrl", + "ServiceName", + "MetricName", + "MetricDescription", + "MetricUnit", + "Attributes", + "StartTimeUnix", + "TimeUnix", + "Count", + "Sum", + "Scale", + "ZeroCount", + "PositiveOffset", + "PositiveBucketCounts", + "NegativeOffset", + "NegativeBucketCounts", + "ExemplarsTraceId", + "ExemplarsSpanId", + "ExemplarsTimestamp", + "ExemplarsValue", + "ExemplarsFilteredAttributes", + "Flags", + "Min", + "Max", + "AggregationTemporality" + ], + "selects": [ + "__ORG__", + "resource_attributes", + "resource_schema_url", + "scope_name", + "scope_version", + "scope_attributes", + "scope_schema_url", + "service_name", + "metric_name", + "metric_description", + "metric_unit", + "metric_attributes", + "start_timestamp", + "timestamp", + "count", + "sum", + "scale", + "zero_count", + "positive_offset", + "positive_bucket_counts", + "negative_offset", + "negative_bucket_counts", + "exemplars_trace_id", + "exemplars_span_id", + "exemplars_timestamp", + "exemplars_value", + "exemplars_filtered_attributes", + "flags", + "min", + "max", + "aggregation_temporality" + ], + "inputSchema": "resource_attributes Map(LowCardinality(String), String), resource_schema_url String, scope_name String, scope_version String, scope_attributes Map(LowCardinality(String), String), scope_schema_url String, service_name LowCardinality(String), metric_name LowCardinality(String), metric_description LowCardinality(String), metric_unit LowCardinality(String), metric_attributes Map(LowCardinality(String), String), start_timestamp DateTime64(9), timestamp DateTime64(9), count UInt64, sum Float64, scale Int32, zero_count UInt64, positive_offset Int32, positive_bucket_counts Array(UInt64), negative_offset Int32, negative_bucket_counts Array(UInt64), exemplars_trace_id Array(String), exemplars_span_id Array(String), exemplars_timestamp Array(DateTime64(9)), exemplars_value Array(Float64), exemplars_filtered_attributes Array(Map(LowCardinality(String), String)), flags UInt32, min Nullable(Float64), max Nullable(Float64), aggregation_temporality Int32" + } + } } diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 3f13c0bb..9e42b98d 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd +-- projectRevision: a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -130,7 +130,12 @@ CREATE TABLE IF NOT EXISTS logs ( ScopeVersion String, ScopeAttributes Map(LowCardinality(String), String), LogAttributes Map(LowCardinality(String), String), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimestampTime) @@ -651,7 +656,9 @@ CREATE TABLE IF NOT EXISTS traces ( INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) @@ -1073,7 +1080,7 @@ SELECT OrgId, toStartOfHour(toDateTime(Timestamp)) AS Hour, ServiceName, - SpanAttributes['db.system.name'] AS DbSystem, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, ResourceAttributes['deployment.environment'] AS DeploymentEnv, count() AS CallCount, countIf(StatusCode = 'Error') AS ErrorCount, @@ -1084,7 +1091,7 @@ SELECT sum(SampleRate) AS SampleRateSum FROM traces WHERE SpanKind IN ('Client', 'Producer') - AND SpanAttributes['db.system.name'] != '' + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' AND ServiceName != '' GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv; diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index d6c8f675..2d280d5c 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,12 +1,12 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd"; +pub const PROJECT_REVISION: &str = "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse // clickHouseSchemaVersion. -pub const SCHEMA_VERSION: &str = "4"; +pub const SCHEMA_VERSION: &str = "5"; pub const ORG_PLACEHOLDER: &str = "__ORG__"; #[derive(Debug)] diff --git a/lib/clickhouse-builder/src/ch/expr.ts b/lib/clickhouse-builder/src/ch/expr.ts index ab773a72..de953d7f 100644 --- a/lib/clickhouse-builder/src/ch/expr.ts +++ b/lib/clickhouse-builder/src/ch/expr.ts @@ -266,6 +266,7 @@ export { left_, length_, lower_, + hasToken, replaceOne, extract_, concat, @@ -294,6 +295,7 @@ export { arrayOf, arrayStringConcat, arrayFilter, + has, mapContains, mapGet, mapKeys, diff --git a/lib/clickhouse-builder/src/ch/functions/array.ts b/lib/clickhouse-builder/src/ch/functions/array.ts index 4f9aff50..6f2bdbe0 100644 --- a/lib/clickhouse-builder/src/ch/functions/array.ts +++ b/lib/clickhouse-builder/src/ch/functions/array.ts @@ -1,4 +1,5 @@ import { makeExpr } from "../expr" +import { defineCondFn } from "../define-fn" import { raw, str, compile } from "../../sql/sql-fragment" import type { Expr } from "../expr" @@ -29,3 +30,10 @@ export function arrayStringConcat( export function arrayFilter(fn: string, arr: Expr): Expr { return makeExpr(raw(`arrayFilter(${fn}, ${compile(arr.toFragment())})`)) } + +/** + * `has(arr, elem)` — true when `elem` is an element of `arr`. Used with a + * `bloom_filter` skip index over a concatenated `key=value` items expression so + * attribute-equality filters prune granules (ClickStack "Items" pattern). + */ +export const has = defineCondFn<[Expr>, Expr]>("has") diff --git a/lib/clickhouse-builder/src/ch/functions/index.ts b/lib/clickhouse-builder/src/ch/functions/index.ts index 5c8337b7..b944d5d1 100644 --- a/lib/clickhouse-builder/src/ch/functions/index.ts +++ b/lib/clickhouse-builder/src/ch/functions/index.ts @@ -27,6 +27,7 @@ export { left_, length_, lower_, + hasToken, replaceOne, extract_, concat, @@ -58,7 +59,7 @@ export { export { if_, multiIf, coalesce, nullIf } from "./conditional" -export { arrayOf, arrayStringConcat, arrayFilter } from "./array" +export { arrayOf, arrayStringConcat, arrayFilter, has } from "./array" export { mapContains, mapGet, mapKeys, mapValues, mapLiteral } from "./map" diff --git a/lib/clickhouse-builder/src/ch/functions/string.ts b/lib/clickhouse-builder/src/ch/functions/string.ts index 31336157..0fc419c8 100644 --- a/lib/clickhouse-builder/src/ch/functions/string.ts +++ b/lib/clickhouse-builder/src/ch/functions/string.ts @@ -1,4 +1,4 @@ -import { defineFn, compileFnCall } from "../define-fn" +import { defineFn, defineCondFn, compileFnCall } from "../define-fn" import type { Expr } from "../expr" // --------------------------------------------------------------------------- @@ -13,6 +13,14 @@ export const positionCaseInsensitive = defineFn<[Expr, Expr], nu ) export const left_ = defineFn<[Expr, Expr], string>("left") +/** + * `hasToken(haystack, token)` — true when `token` occurs as a whole token of + * `haystack` (separators are non-alphanumeric ASCII). Backed by `tokenbf_v1` + * skip indexes for granule pruning. Whole-token semantics ≠ substring `ILIKE`; + * see `body-search.ts` in the query engine for the safe-token derivation. + */ +export const hasToken = defineCondFn<[Expr, Expr]>("hasToken") + // --------------------------------------------------------------------------- // Mixed Expr + literal args (compileFnCall wrappers) // --------------------------------------------------------------------------- diff --git a/lib/clickhouse-builder/src/ch/index.ts b/lib/clickhouse-builder/src/ch/index.ts index 1cf23845..7386245d 100644 --- a/lib/clickhouse-builder/src/ch/index.ts +++ b/lib/clickhouse-builder/src/ch/index.ts @@ -76,6 +76,8 @@ export { position_ as position, left_ as left, length_ as length, + lower_ as lower, + hasToken, replaceOne, extract_ as extract, concat, @@ -107,6 +109,7 @@ export { arrayOf, arrayStringConcat, arrayFilter, + has, // Map mapContains, mapGet, diff --git a/packages/domain/package.json b/packages/domain/package.json index 5d6e88be..c6b2b899 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -15,6 +15,7 @@ "./warehouse-queries": "./src/warehouse-queries.ts", "./tinybird": "./src/tinybird/index.ts", "./tinybird/db-query-shape-sql": "./src/tinybird/db-query-shape-sql.ts", + "./tinybird/index-exprs": "./src/tinybird/index-exprs.ts", "./clickhouse": "./src/clickhouse/index.ts", "./where-clause": "./src/where-clause.ts", "./http/observability": "./src/http/observability.ts", diff --git a/packages/domain/src/clickhouse/migrations/0005_body_and_attr_indexes.ts b/packages/domain/src/clickhouse/migrations/0005_body_and_attr_indexes.ts new file mode 100644 index 00000000..8d9e95f7 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0005_body_and_attr_indexes.ts @@ -0,0 +1,49 @@ +import { attrItemsExpr, BODY_TOKEN_INDEX_EXPR } from "../../tinybird/index-exprs" + +/** + * Migration 0005 — full-text body index + attribute skip indexes. + * + * Adds ClickStack-style data-skipping indexes (see "Making ClickStack 5x + * faster"): + * + * - `logs.idx_body_tokens` — `tokenbf_v1` over `lower(Body)` so the query + * engine's `hasToken(lower(Body), )` pre-filter prunes granules + * ahead of the substring `Body ILIKE '%…%'`. + * - `logs.idx_log_*` — bloom filters over `LogAttributes`/`ResourceAttributes` + * keys and values, mirroring the ones `traces` already has. + * - `traces.idx_*_attr_items` — bloom filters over the concatenated + * `key=value` "Items" expression so attribute-equality filters prune via + * `has(, 'k=v')`. + * + * The index expressions are imported from the same {@link attrItemsExpr} / + * {@link BODY_TOKEN_INDEX_EXPR} helpers the datasource definitions and the + * query-engine predicates use — the three must stay byte-for-byte identical or + * ClickHouse silently stops applying the index. + * + * Deliberately **no `MATERIALIZE INDEX`**: materializing over an existing + * multi-billion-row part set exceeds the Cloudflare Worker subrequest budget + * (same reason migration 0004 adds its `set` index without materializing). The + * indexes apply to new parts immediately; with the 30-day TTL the whole table + * is covered within a retention cycle as old parts roll off. An operator who + * wants immediate coverage can run, partition by partition: + * + * ALTER TABLE logs MATERIALIZE INDEX idx_body_tokens IN PARTITION '' + * ALTER TABLE traces MATERIALIZE INDEX idx_span_attr_items IN PARTITION '' + * + * All statements are `IF NOT EXISTS`, so they're no-ops on a fresh instance + * whose migration-0001 snapshot already created the indexes at CREATE TABLE. + */ +export const migration_0005_body_and_attr_indexes = { + version: 5, + description: + "Add tokenbf body index + logs attribute blooms + traces attribute 'items' blooms for search/filter granule pruning", + statements: [ + `ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_body_tokens ${BODY_TOKEN_INDEX_EXPR} TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1`, + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", + `ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_span_attr_items ${attrItemsExpr("SpanAttributes")} TYPE bloom_filter(0.01) GRANULARITY 1`, + `ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_resource_attr_items ${attrItemsExpr("ResourceAttributes")} TYPE bloom_filter(0.01) GRANULARITY 1`, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 89e2c669..9941cf88 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest" import { isBackfill, renderStatementFull, type BackfillSpec } from "../backfill" import { migration_0004_service_namespace_projections } from "./0004_service_namespace_projections" +import { migration_0005_body_and_attr_indexes } from "./0005_body_and_attr_indexes" import { migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -14,9 +15,22 @@ const renderedSql = migration_0004_service_namespace_projections.statements .join("\n\n") describe("ClickHouse migrations", () => { - it("keeps service.namespace migration ordered after the previous deltas", () => { - expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4]) - expect(migrations.at(-1)).toBe(migration_0004_service_namespace_projections) + it("keeps migrations ordered with the body/attr-index delta last", () => { + expect(migrations.map((m) => m.version)).toEqual([1, 2, 3, 4, 5]) + expect(migrations.at(-1)).toBe(migration_0005_body_and_attr_indexes) + }) + + it("adds body + attribute skip indexes as new-parts-only ADD INDEX statements", () => { + const sql = migration_0005_body_and_attr_indexes.statements.join("\n") + expect(sql).toContain( + "ALTER TABLE logs ADD INDEX IF NOT EXISTS idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1", + ) + expect(sql).toContain( + "ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1", + ) + // Must NOT materialize — that would exceed the Worker subrequest budget on + // billion-row parts; indexes apply to new parts and backfill via TTL churn. + expect(sql).not.toContain("MATERIALIZE INDEX") }) it("rebuilds namespace-aware log aggregates and recreates affected materialized views", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 226f6a77..1e0a6044 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -3,6 +3,7 @@ import { migration_0001_initial } from "./0001_initial" import { migration_0002_service_map_edges_rollup } from "./0002_service_map_edges_rollup" import { migration_0003_error_events_label } from "./0003_error_events_label" import { migration_0004_service_namespace_projections } from "./0004_service_namespace_projections" +import { migration_0005_body_and_attr_indexes } from "./0005_body_and_attr_indexes" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -36,6 +37,7 @@ export const migrations: ReadonlyArray = [ migration_0002_service_map_edges_rollup, migration_0003_error_events_label, migration_0004_service_namespace_projections, + migration_0005_body_and_attr_indexes, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 312d93cc..23bc1673 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "d58ce4a83d3ad3f3a29b9bb972272b757547ae793c050194354454634f3abccd" as const +export const projectRevision = "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 90 DAY", @@ -10,7 +10,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS error_events (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, FingerprintHash, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_events_by_time (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, FingerprintHash)\nTTL Timestamp + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS error_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, Timestamp)\nTTL Timestamp + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS logs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TimestampTime DateTime,\n TraceId String,\n SpanId String,\n TraceFlags UInt8,\n SeverityText LowCardinality(String),\n SeverityNumber UInt8,\n ServiceName LowCardinality(String),\n Body String,\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n LogAttributes Map(LowCardinality(String), String),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimestampTime)\nORDER BY (OrgId, ServiceName, TimestampTime, Timestamp)\nTTL toDate(TimestampTime) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS logs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TimestampTime DateTime,\n TraceId String,\n SpanId String,\n TraceFlags UInt8,\n SeverityText LowCardinality(String),\n SeverityNumber UInt8,\n ServiceName LowCardinality(String),\n Body String,\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n LogAttributes Map(LowCardinality(String), String),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,\n INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimestampTime)\nORDER BY (OrgId, ServiceName, TimestampTime, Timestamp)\nTTL toDate(TimestampTime) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS logs_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metric_catalog (\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour)\nTTL Hour + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_exponential_histogram (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Count UInt64,\n Sum Float64,\n Scale Int32,\n ZeroCount UInt64,\n PositiveOffset Int32,\n PositiveBucketCounts Array(UInt64),\n NegativeOffset Int32,\n NegativeBucketCounts Array(UInt64),\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n Flags UInt32,\n Min Nullable(Float64),\n Max Nullable(Float64),\n AggregationTemporality Int32\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", @@ -33,7 +33,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS trace_detail_spans (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, SpanId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS trace_list_mv (\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String),\n INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL Timestamp + INTERVAL 30 DAY", - "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", + "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'", @@ -49,7 +49,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", - "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanAttributes['db.system.name'] AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n)))\n) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv, QueryKey", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 99e020eb..0e89cc4d 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,336 +1,272 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "5e5eac5fd0008ab8726225d6f6182c1855be4d8620aedea83b4f83d0b78d8278" as const +export const projectRevision = "a8d56c30f4f7c8998db16ca014680db16a82f69143adbeb054ccc47d718b222b" as const export const datasources = [ - { - name: "alert_checks", - content: - 'DESCRIPTION >\n One row per alert rule evaluation. Durable audit trail of checks: status, observed value, threshold, sample count, incident linkage.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n RuleId String `json:$.RuleId`,\n GroupKey String `json:$.GroupKey`,\n Timestamp DateTime64(3) `json:$.Timestamp`,\n Status LowCardinality(String) `json:$.Status`,\n SignalType LowCardinality(String) `json:$.SignalType`,\n Comparator LowCardinality(String) `json:$.Comparator`,\n Threshold Float64 `json:$.Threshold`,\n ObservedValue Nullable(Float64) `json:$.ObservedValue`,\n SampleCount UInt32 `json:$.SampleCount`,\n WindowMinutes UInt16 `json:$.WindowMinutes`,\n WindowStart DateTime64(3) `json:$.WindowStart`,\n WindowEnd DateTime64(3) `json:$.WindowEnd`,\n ConsecutiveBreaches UInt16 `json:$.ConsecutiveBreaches`,\n ConsecutiveHealthy UInt16 `json:$.ConsecutiveHealthy`,\n IncidentId Nullable(String) `json:$.IncidentId`,\n IncidentTransition LowCardinality(String) `json:$.IncidentTransition`,\n EvaluationDurationMs UInt32 `json:$.EvaluationDurationMs`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, RuleId, GroupKey, Timestamp"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 90 DAY"', - }, - { - name: "attribute_keys_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated attribute keys with hourly usage counts from traces, logs, and metrics.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, AttributeScope, Hour, AttributeKey"\nENGINE_TTL "Hour + INTERVAL 90 DAY"', - }, - { - name: "attribute_values_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated attribute values with hourly usage counts from trace span and resource attributes.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, AttributeScope, AttributeKey, Hour, AttributeValue"\nENGINE_TTL "Hour + INTERVAL 90 DAY"', - }, - { - name: "error_events", - content: - 'DESCRIPTION >\n Per-error-occurrence rows for the triageable-errors system. Unwraps OTel exception events and computes a stable FingerprintHash for grouping into issues. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, FingerprintHash, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', - }, - { - name: "error_events_by_time", - content: - 'DESCRIPTION >\n Time-ordered sibling of error_events (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans (errorIssuesScan tick + dashboard error queries). Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, Timestamp, FingerprintHash"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', - }, - { - name: "error_spans", - content: - 'DESCRIPTION >\n Pre-materialized error spans for the errors page. Pre-filters to StatusCode=\'Error\' and pre-extracts deployment.environment. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT \'__unset__\',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, ServiceName, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 90 DAY"', - }, - { - name: "logs", - content: - 'DESCRIPTION >\n This is a table that contains the logs from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n TimestampTime DateTime `json:$.timestamp`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n TraceFlags UInt8 `json:$.flags`,\n SeverityText LowCardinality(String) `json:$.severity_text`,\n SeverityNumber UInt8 `json:$.severity_number`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n Body String `json:$.body`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n LogAttributes Map(LowCardinality(String), String) `json:$.log_attributes`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimestampTime)"\nENGINE_SORTING_KEY "OrgId, ServiceName, TimestampTime, Timestamp"\nENGINE_TTL "toDate(TimestampTime) + INTERVAL 30 DAY"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1', - }, - { - name: "logs_aggregates_hourly", - content: - 'DESCRIPTION >\n Hourly pre-aggregated log counts and sizes by service × severity × deployment env. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, SeverityText, DeploymentEnv,\n Count, SizeBytes,\n ServiceNamespace', - }, - { - name: "metric_catalog", - content: - 'DESCRIPTION >\n Hourly catalog of distinct metrics (name/type/service) with datapoint counts and first/last-seen. AggregatingMergeTree MV target; powers the Metrics page discovery queries.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, MetricType, ServiceName, MetricName, Hour"\nENGINE_TTL "Hour + INTERVAL 90 DAY"', - }, - { - name: "metrics_exponential_histogram", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n Scale Int32 `json:$.scale`,\n ZeroCount UInt64 `json:$.zero_count`,\n PositiveOffset Int32 `json:$.positive_offset`,\n PositiveBucketCounts Array(UInt64) `json:$.positive_bucket_counts[:]`,\n NegativeOffset Int32 `json:$.negative_offset`,\n NegativeBucketCounts Array(UInt64) `json:$.negative_bucket_counts[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "metrics_gauge", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "metrics_histogram", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n BucketCounts Array(UInt64) `json:$.bucket_counts[:]`,\n ExplicitBounds Array(Float64) `json:$.explicit_bounds[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "metrics_sum", - content: - 'DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`,\n IsMonotonic Bool `json:$.is_monotonic`\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(TimeUnix)"\nENGINE_SORTING_KEY "OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)"\nENGINE_TTL "toDate(TimeUnix) + INTERVAL 90 DAY"', - }, - { - name: "service_address_resolutions_hourly", - content: - 'DESCRIPTION >\n Resolved (sourceService, parent.server.address) → resolved targetService facts emitted by the ServiceMapRollupService rollup. Used to anti-join internal-service overlap out of the external-edges query.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "service_external_edges_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab. Captures Client/Producer spans WITHOUT db.system.name. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "service_map_children", - content: - 'DESCRIPTION >\n Server/Consumer spans with ParentSpanId for efficient service map child-side JOIN lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, ParentSpanId, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"', - }, - { - name: "service_map_db_edges_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly service-to-database edges (one row per service/db.system.name) for the service map\'s database-node query. Uses AggregatingMergeTree for incremental aggregation. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, ServiceName, DbSystem"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT *', - }, - { - name: "service_map_db_query_shapes_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly database query shapes (one row per service/db.system/query-shape) for the service map\'s database detail panel. Uses AggregatingMergeTree with a sample-weighted t-digest state for true p50/p95. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n QueryKey String,\n QueryLabel SimpleAggregateFunction(any, String),\n SampleStatement SimpleAggregateFunction(any, String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedCount SimpleAggregateFunction(sum, Float64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSumMs SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, QueryKey"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT *', - }, - { - name: "service_map_edges_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly service-to-service edges for the service map. Uses AggregatingMergeTree for incremental aggregation. Populated by the scheduled ServiceMapRollupService rollup (one write per completed hour).\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n TargetService String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, DeploymentEnv, SourceService, TargetService"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "service_map_spans", - content: - 'DESCRIPTION >\n Lightweight projection of traces for service map JOIN queries. Pre-extracts deployment.environment from Map columns. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, SpanId, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"', - }, - { - name: "service_overview_spans", - content: - 'DESCRIPTION >\n Lightweight projection of service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, ServiceName, Timestamp"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4', - }, - { - name: "service_platforms_hourly", - content: - 'DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) for the service map\'s hosting-icon resolver. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, DeploymentEnv,\n K8sCluster, K8sPodName, K8sDeploymentName,\n K8sStatefulSetName,\n K8sDaemonSetName,\n K8sNamespaceName,\n CloudPlatform, CloudProvider, FaasName, MapleSdkType, ProcessRuntimeName, SpanCount', - }, - { - name: "service_usage", - content: - 'DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n\nENGINE "SummingMergeTree"\nENGINE_SORTING_KEY "OrgId, ServiceName, Hour"\nENGINE_TTL "Hour + INTERVAL 365 DAY"\n\nFORWARD_QUERY >\n SELECT *', - }, - { - name: "session_events", - content: - "DESCRIPTION >\n Distilled structured session events (navigation, click, input, console, network, error) captured client-side and ingested via POST /v1/sessionEvents. Powers in-session search, replay panels, and agent transcripts.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Seq UInt32 `json:$.seq` DEFAULT 0,\n Type LowCardinality(String) `json:$.type`,\n Url String `json:$.url` DEFAULT '',\n TraceId String `json:$.trace_id` DEFAULT '',\n Level LowCardinality(String) `json:$.level` DEFAULT '',\n Message String `json:$.message` DEFAULT '',\n TargetSelector String `json:$.target_selector` DEFAULT '',\n TargetText String `json:$.target_text` DEFAULT '',\n NetMethod LowCardinality(String) `json:$.net_method` DEFAULT '',\n NetUrl String `json:$.net_url` DEFAULT '',\n NetStatus UInt16 `json:$.net_status` DEFAULT 0,\n NetDurationMs UInt32 `json:$.net_duration_ms` DEFAULT 0,\n ErrorStack String `json:$.error_stack` DEFAULT '',\n Attributes Map(LowCardinality(String), String) `json:$.attributes`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, Timestamp, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"", - }, - { - name: "session_replay_events", - content: - 'DESCRIPTION >\n Session replay rrweb events (one row per chunk, payload included). The ingest gateway gunzips the chunk and stores the event-array JSON in `Events`. Playback reads directly from ClickHouse — no R2.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n ChunkSeq UInt32 `json:$.chunk_seq`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n DurationMs UInt32 `json:$.duration_ms` DEFAULT 0,\n EventCount UInt32 `json:$.event_count` DEFAULT 0,\n ByteSize UInt32 `json:$.byte_size` DEFAULT 0,\n Events String `json:$.events`,\n IsCheckpoint UInt8 `json:$.is_checkpoint` DEFAULT 0\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, SessionId, ChunkSeq"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"', - }, - { - name: "session_replays", - content: - 'DESCRIPTION >\n Per-session browser replay metadata (one row per session). Ingested directly from the @maple-dev/browser SDK via POST /v1/sessionReplays/meta. Event payloads live inline in session_replay_events; this holds only queryable metadata. ReplacingMergeTree(Version) for start/end upsert.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n StartTime DateTime64(9) `json:$.start_time`,\n EndTime Nullable(DateTime64(9)) `json:$.end_time`,\n DurationMs Nullable(UInt32) `json:$.duration_ms`,\n Status LowCardinality(String) `json:$.status`,\n UserId String `json:$.user_id`,\n UrlInitial String `json:$.url_initial`,\n UserAgent String `json:$.user_agent`,\n BrowserName LowCardinality(String) `json:$.browser_name`,\n OsName LowCardinality(String) `json:$.os_name`,\n DeviceType LowCardinality(String) `json:$.device_type`,\n Country LowCardinality(String) `json:$.country` DEFAULT \'\',\n ServiceName LowCardinality(String) `json:$.service_name`,\n PageViews UInt32 `json:$.page_views` DEFAULT 0,\n ClickCount UInt32 `json:$.click_count` DEFAULT 0,\n ErrorCount UInt32 `json:$.error_count` DEFAULT 0,\n TraceIds Array(String) `json:$.trace_ids[:]` DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n Version UInt32 `json:$.version`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toDate(StartTime)"\nENGINE_SORTING_KEY "OrgId, SessionId"\nENGINE_TTL "toDate(StartTime) + INTERVAL 30 DAY"\nENGINE_VER "Version"', - }, - { - name: "span_metrics_calls_hourly", - content: - 'DESCRIPTION >\n Hourly per-series last-value (argMax) rollup of the span-metrics calls counter. AggregatingMergeTree MV target powering sampling-aware throughput without scanning raw metrics_sum.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, - { - name: "trace_detail_spans", - content: - 'DESCRIPTION >\n All spans for a trace, sorted by TraceId for fast detail lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, TraceId, SpanId"\nENGINE_TTL "toDate(Timestamp) + INTERVAL 30 DAY"', - }, - { - name: "trace_list_mv", - content: - 'DESCRIPTION >\n Pre-materialized root spans for the trace list view. Extracts HTTP attributes and normalizes span names at write time. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String)\n\nENGINE "MergeTree"\nENGINE_PARTITION_KEY "toDate(Timestamp)"\nENGINE_SORTING_KEY "OrgId, Timestamp, TraceId"\nENGINE_TTL "Timestamp + INTERVAL 30 DAY"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4', - }, - { - name: "traces", - content: - "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1", - }, - { - name: "traces_aggregates_hourly", - content: - 'DESCRIPTION >\n Hourly pre-aggregated trace metrics with sampling-weighted state columns. Generalized MV target for timeseries/breakdown/service-overview queries. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n\nENGINE "AggregatingMergeTree"\nENGINE_PARTITION_KEY "toDate(Hour)"\nENGINE_SORTING_KEY "OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv"\nENGINE_TTL "toDate(Hour) + INTERVAL 90 DAY"', - }, + { + "name": "alert_checks", + "content": "DESCRIPTION >\n One row per alert rule evaluation. Durable audit trail of checks: status, observed value, threshold, sample count, incident linkage.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n RuleId String `json:$.RuleId`,\n GroupKey String `json:$.GroupKey`,\n Timestamp DateTime64(3) `json:$.Timestamp`,\n Status LowCardinality(String) `json:$.Status`,\n SignalType LowCardinality(String) `json:$.SignalType`,\n Comparator LowCardinality(String) `json:$.Comparator`,\n Threshold Float64 `json:$.Threshold`,\n ObservedValue Nullable(Float64) `json:$.ObservedValue`,\n SampleCount UInt32 `json:$.SampleCount`,\n WindowMinutes UInt16 `json:$.WindowMinutes`,\n WindowStart DateTime64(3) `json:$.WindowStart`,\n WindowEnd DateTime64(3) `json:$.WindowEnd`,\n ConsecutiveBreaches UInt16 `json:$.ConsecutiveBreaches`,\n ConsecutiveHealthy UInt16 `json:$.ConsecutiveHealthy`,\n IncidentId Nullable(String) `json:$.IncidentId`,\n IncidentTransition LowCardinality(String) `json:$.IncidentTransition`,\n EvaluationDurationMs UInt32 `json:$.EvaluationDurationMs`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, RuleId, GroupKey, Timestamp\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 90 DAY\"" + }, + { + "name": "attribute_keys_hourly", + "content": "DESCRIPTION >\n Pre-aggregated attribute keys with hourly usage counts from traces, logs, and metrics.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, AttributeScope, Hour, AttributeKey\"\nENGINE_TTL \"Hour + INTERVAL 90 DAY\"" + }, + { + "name": "attribute_values_hourly", + "content": "DESCRIPTION >\n Pre-aggregated attribute values with hourly usage counts from trace span and resource attributes.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, AttributeScope, AttributeKey, Hour, AttributeValue\"\nENGINE_TTL \"Hour + INTERVAL 90 DAY\"" + }, + { + "name": "error_events", + "content": "DESCRIPTION >\n Per-error-occurrence rows for the triageable-errors system. Unwraps OTel exception events and computes a stable FingerprintHash for grouping into issues. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, FingerprintHash, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 90 DAY\"" + }, + { + "name": "error_events_by_time", + "content": "DESCRIPTION >\n Time-ordered sibling of error_events (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans (errorIssuesScan tick + dashboard error queries). Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n ExceptionType LowCardinality(String),\n ExceptionMessage String,\n ExceptionStacktrace String,\n TopFrame String,\n FingerprintHash UInt64,\n StatusMessage String,\n Duration UInt64,\n ErrorLabel String\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, FingerprintHash\"\nENGINE_TTL \"Timestamp + INTERVAL 90 DAY\"" + }, + { + "name": "error_spans", + "content": "DESCRIPTION >\n Pre-materialized error spans for the errors page. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String DEFAULT '__unset__',\n ServiceName LowCardinality(String),\n StatusMessage String,\n Duration UInt64,\n DeploymentEnv LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 90 DAY\"" + }, + { + "name": "logs", + "content": "DESCRIPTION >\n This is a table that contains the logs from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n TimestampTime DateTime `json:$.timestamp`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n TraceFlags UInt8 `json:$.flags`,\n SeverityText LowCardinality(String) `json:$.severity_text`,\n SeverityNumber UInt8 `json:$.severity_number`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n Body String `json:$.body`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n LogAttributes Map(LowCardinality(String), String) `json:$.log_attributes`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimestampTime)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, TimestampTime, Timestamp\"\nENGINE_TTL \"toDate(TimestampTime) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_body_tokens lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1\n idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_log_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_log_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1" + }, + { + "name": "logs_aggregates_hourly", + "content": "DESCRIPTION >\n Hourly pre-aggregated log counts and sizes by service × severity × deployment env. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SeverityText LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n Count SimpleAggregateFunction(sum, UInt64),\n SizeBytes SimpleAggregateFunction(sum, UInt64),\n ServiceNamespace LowCardinality(String)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, SeverityText, DeploymentEnv,\n Count, SizeBytes,\n ServiceNamespace" + }, + { + "name": "metric_catalog", + "content": "DESCRIPTION >\n Hourly catalog of distinct metrics (name/type/service) with datapoint counts and first/last-seen. AggregatingMergeTree MV target; powers the Metrics page discovery queries.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n MetricType LowCardinality(String),\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription SimpleAggregateFunction(anyLast, String),\n MetricUnit SimpleAggregateFunction(anyLast, String),\n IsMonotonic SimpleAggregateFunction(anyLast, UInt8),\n DataPointCount SimpleAggregateFunction(sum, UInt64),\n FirstSeen SimpleAggregateFunction(min, DateTime),\n LastSeen SimpleAggregateFunction(max, DateTime)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, MetricType, ServiceName, MetricName, Hour\"\nENGINE_TTL \"Hour + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_exponential_histogram", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n Scale Int32 `json:$.scale`,\n ZeroCount UInt64 `json:$.zero_count`,\n PositiveOffset Int32 `json:$.positive_offset`,\n PositiveBucketCounts Array(UInt64) `json:$.positive_bucket_counts[:]`,\n NegativeOffset Int32 `json:$.negative_offset`,\n NegativeBucketCounts Array(UInt64) `json:$.negative_bucket_counts[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_gauge", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_histogram", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Count UInt64 `json:$.count`,\n Sum Float64 `json:$.sum`,\n BucketCounts Array(UInt64) `json:$.bucket_counts[:]`,\n ExplicitBounds Array(Float64) `json:$.explicit_bounds[:]`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n Flags UInt32 `json:$.flags`,\n Min Nullable(Float64) `json:$.min`,\n Max Nullable(Float64) `json:$.max`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "metrics_sum", + "content": "DESCRIPTION >\n This is a table that contains the metrics from the OpenTelemetry Collector.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n MetricName LowCardinality(String) `json:$.metric_name`,\n MetricDescription LowCardinality(String) `json:$.metric_description`,\n MetricUnit LowCardinality(String) `json:$.metric_unit`,\n Attributes Map(LowCardinality(String), String) `json:$.metric_attributes`,\n StartTimeUnix DateTime64(9) `json:$.start_timestamp`,\n TimeUnix DateTime64(9) `json:$.timestamp`,\n Value Float64 `json:$.value`,\n Flags UInt32 `json:$.flags`,\n ExemplarsTraceId Array(String) `json:$.exemplars_trace_id[:]`,\n ExemplarsSpanId Array(String) `json:$.exemplars_span_id[:]`,\n ExemplarsTimestamp Array(DateTime64(9)) `json:$.exemplars_timestamp[:]`,\n ExemplarsValue Array(Float64) `json:$.exemplars_value[:]`,\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) `json:$.exemplars_filtered_attributes[:]`,\n AggregationTemporality Int32 `json:$.aggregation_temporality`,\n IsMonotonic Bool `json:$.is_monotonic`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(TimeUnix)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)\"\nENGINE_TTL \"toDate(TimeUnix) + INTERVAL 90 DAY\"" + }, + { + "name": "service_address_resolutions_hourly", + "content": "DESCRIPTION >\n Resolved (sourceService, parent.server.address) → resolved targetService facts emitted by the ServiceMapRollupService rollup. Used to anti-join internal-service overlap out of the external-edges query.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n\nENGINE \"ReplacingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "service_external_edges_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab. Captures Client/Producer spans WITHOUT db.system.name. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "service_map_children", + "content": "DESCRIPTION >\n Server/Consumer spans with ParentSpanId for efficient service map child-side JOIN lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, TraceId, ParentSpanId, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"" + }, + { + "name": "service_map_db_edges_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly service-to-database edges (one row per service/db.system.name) for the service map's database-node query. Uses AggregatingMergeTree for incremental aggregation. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, ServiceName, DbSystem\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nFORWARD_QUERY >\n SELECT *" + }, + { + "name": "service_map_db_query_shapes_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly database query shapes (one row per service/db.system/query-shape) for the service map's database detail panel. Uses AggregatingMergeTree with a sample-weighted t-digest state for true p50/p95. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DbSystem LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n QueryKey String,\n QueryLabel SimpleAggregateFunction(any, String),\n SampleStatement SimpleAggregateFunction(any, String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n EstimatedCount SimpleAggregateFunction(sum, Float64),\n EstimatedErrorCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSumMs SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, QueryKey\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nFORWARD_QUERY >\n SELECT *" + }, + { + "name": "service_map_edges_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly service-to-service edges for the service map. Uses AggregatingMergeTree for incremental aggregation. Populated by the scheduled ServiceMapRollupService rollup (one write per completed hour).\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n TargetService String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampledSpanCount SimpleAggregateFunction(sum, UInt64),\n UnsampledSpanCount SimpleAggregateFunction(sum, UInt64),\n SampleRateSum SimpleAggregateFunction(sum, Float64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, DeploymentEnv, SourceService, TargetService\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "service_map_spans", + "content": "DESCRIPTION >\n Lightweight projection of traces for service map JOIN queries. Pre-extracts deployment.environment from Map columns. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n SpanId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, TraceId, SpanId, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"" + }, + { + "name": "service_overview_spans", + "content": "DESCRIPTION >\n Lightweight projection of service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String),\n CommitSha LowCardinality(String),\n SampleRate Float64 DEFAULT 1,\n ServiceNamespace LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, Timestamp\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4" + }, + { + "name": "service_platforms_hourly", + "content": "DESCRIPTION >\n Pre-aggregated hourly per-service platform/runtime attributes (k8s, cloud, faas) for the service map's hosting-icon resolver. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n K8sCluster SimpleAggregateFunction(max, String),\n K8sPodName SimpleAggregateFunction(max, String),\n K8sDeploymentName SimpleAggregateFunction(max, String),\n K8sStatefulSetName SimpleAggregateFunction(max, String),\n K8sDaemonSetName SimpleAggregateFunction(max, String),\n K8sNamespaceName SimpleAggregateFunction(max, String),\n CloudPlatform SimpleAggregateFunction(max, String),\n CloudProvider SimpleAggregateFunction(max, String),\n FaasName SimpleAggregateFunction(max, String),\n MapleSdkType SimpleAggregateFunction(max, String),\n ProcessRuntimeName SimpleAggregateFunction(max, String),\n SpanCount SimpleAggregateFunction(sum, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, DeploymentEnv\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"\n\nFORWARD_QUERY >\n SELECT\n OrgId, Hour, ServiceName, DeploymentEnv,\n K8sCluster, K8sPodName, K8sDeploymentName,\n K8sStatefulSetName,\n K8sDaemonSetName,\n K8sNamespaceName,\n CloudPlatform, CloudProvider, FaasName, MapleSdkType, ProcessRuntimeName, SpanCount" + }, + { + "name": "service_usage", + "content": "DESCRIPTION >\n Aggregated usage statistics per service per hour. Uses SummingMergeTree for efficient incremental updates from multiple materialized views.\n\nSCHEMA >\n OrgId LowCardinality(String),\n ServiceName LowCardinality(String),\n Hour DateTime,\n LogCount UInt64,\n LogSizeBytes UInt64,\n TraceCount UInt64,\n TraceSizeBytes UInt64,\n SumMetricCount UInt64,\n SumMetricSizeBytes UInt64,\n GaugeMetricCount UInt64,\n GaugeMetricSizeBytes UInt64,\n HistogramMetricCount UInt64,\n HistogramMetricSizeBytes UInt64,\n ExpHistogramMetricCount UInt64,\n ExpHistogramMetricSizeBytes UInt64\n\nENGINE \"SummingMergeTree\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, Hour\"\nENGINE_TTL \"Hour + INTERVAL 365 DAY\"\n\nFORWARD_QUERY >\n SELECT *" + }, + { + "name": "session_events", + "content": "DESCRIPTION >\n Distilled structured session events (navigation, click, input, console, network, error) captured client-side and ingested via POST /v1/sessionEvents. Powers in-session search, replay panels, and agent transcripts.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Seq UInt32 `json:$.seq` DEFAULT 0,\n Type LowCardinality(String) `json:$.type`,\n Url String `json:$.url` DEFAULT '',\n TraceId String `json:$.trace_id` DEFAULT '',\n Level LowCardinality(String) `json:$.level` DEFAULT '',\n Message String `json:$.message` DEFAULT '',\n TargetSelector String `json:$.target_selector` DEFAULT '',\n TargetText String `json:$.target_text` DEFAULT '',\n NetMethod LowCardinality(String) `json:$.net_method` DEFAULT '',\n NetUrl String `json:$.net_url` DEFAULT '',\n NetStatus UInt16 `json:$.net_status` DEFAULT 0,\n NetDurationMs UInt32 `json:$.net_duration_ms` DEFAULT 0,\n ErrorStack String `json:$.error_stack` DEFAULT '',\n Attributes Map(LowCardinality(String), String) `json:$.attributes`\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, Timestamp, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"" + }, + { + "name": "session_replay_events", + "content": "DESCRIPTION >\n Session replay rrweb events (one row per chunk, payload included). The ingest gateway gunzips the chunk and stores the event-array JSON in `Events`. Playback reads directly from ClickHouse — no R2.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n ChunkSeq UInt32 `json:$.chunk_seq`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n DurationMs UInt32 `json:$.duration_ms` DEFAULT 0,\n EventCount UInt32 `json:$.event_count` DEFAULT 0,\n ByteSize UInt32 `json:$.byte_size` DEFAULT 0,\n Events String `json:$.events`,\n IsCheckpoint UInt8 `json:$.is_checkpoint` DEFAULT 0\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, SessionId, ChunkSeq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"" + }, + { + "name": "session_replays", + "content": "DESCRIPTION >\n Per-session browser replay metadata (one row per session). Ingested directly from the @maple-dev/browser SDK via POST /v1/sessionReplays/meta. Event payloads live inline in session_replay_events; this holds only queryable metadata. ReplacingMergeTree(Version) for start/end upsert.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n SessionId String `json:$.session_id`,\n StartTime DateTime64(9) `json:$.start_time`,\n EndTime Nullable(DateTime64(9)) `json:$.end_time`,\n DurationMs Nullable(UInt32) `json:$.duration_ms`,\n Status LowCardinality(String) `json:$.status`,\n UserId String `json:$.user_id`,\n UrlInitial String `json:$.url_initial`,\n UserAgent String `json:$.user_agent`,\n BrowserName LowCardinality(String) `json:$.browser_name`,\n OsName LowCardinality(String) `json:$.os_name`,\n DeviceType LowCardinality(String) `json:$.device_type`,\n Country LowCardinality(String) `json:$.country` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name`,\n PageViews UInt32 `json:$.page_views` DEFAULT 0,\n ClickCount UInt32 `json:$.click_count` DEFAULT 0,\n ErrorCount UInt32 `json:$.error_count` DEFAULT 0,\n TraceIds Array(String) `json:$.trace_ids[:]` DEFAULT [],\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n Version UInt32 `json:$.version`\n\nENGINE \"ReplacingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(StartTime)\"\nENGINE_SORTING_KEY \"OrgId, SessionId\"\nENGINE_TTL \"toDate(StartTime) + INTERVAL 30 DAY\"\nENGINE_VER \"Version\"" + }, + { + "name": "span_metrics_calls_hourly", + "content": "DESCRIPTION >\n Hourly per-series last-value (argMax) rollup of the span-metrics calls counter. AggregatingMergeTree MV target powering sampling-aware throughput without scanning raw metrics_sum.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n SpanKind LowCardinality(String),\n AttrFingerprint UInt64,\n ResourceFingerprint UInt64,\n StartTimeUnix DateTime64(9),\n LastValue AggregateFunction(argMax, Float64, DateTime64(9))\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + }, + { + "name": "trace_detail_spans", + "content": "DESCRIPTION >\n All spans for a trace, sorted by TraceId for fast detail lookups. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n ResourceAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String))\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, TraceId, SpanId\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"" + }, + { + "name": "trace_list_mv", + "content": "DESCRIPTION >\n Pre-materialized root spans for the trace list view. Extracts HTTP attributes and normalizes span names at write time. Populated by materialized view.\n\nSCHEMA >\n OrgId LowCardinality(String),\n TraceId String,\n Timestamp DateTime,\n ServiceName LowCardinality(String),\n SpanName String,\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n HttpMethod LowCardinality(String),\n HttpRoute String,\n HttpStatusCode LowCardinality(String),\n DeploymentEnv LowCardinality(String),\n HasError UInt8,\n TraceState String,\n ServiceNamespace LowCardinality(String)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, TraceId\"\nENGINE_TTL \"Timestamp + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4" + }, + { + "name": "traces", + "content": "DESCRIPTION >\n A table that contains trace data from OpenTelemetry in Tinybird format.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.resource_attributes.maple_org_id`,\n Timestamp DateTime64(9) `json:$.start_time`,\n TraceId String `json:$.trace_id`,\n SpanId String `json:$.span_id`,\n ParentSpanId String `json:$.parent_span_id`,\n TraceState String `json:$.trace_state`,\n SpanName LowCardinality(String) `json:$.span_name`,\n SpanKind LowCardinality(String) `json:$.span_kind`,\n ServiceName LowCardinality(String) `json:$.service_name`,\n ResourceSchemaUrl String `json:$.resource_schema_url`,\n ResourceAttributes Map(LowCardinality(String), String) `json:$.resource_attributes`,\n ScopeSchemaUrl String `json:$.scope_schema_url`,\n ScopeName String `json:$.scope_name`,\n ScopeVersion String `json:$.scope_version`,\n ScopeAttributes Map(LowCardinality(String), String) `json:$.scope_attributes`,\n Duration UInt64 `json:$.duration` DEFAULT 0,\n StatusCode LowCardinality(String) `json:$.status_code`,\n StatusMessage String `json:$.status_message`,\n SpanAttributes Map(LowCardinality(String), String) `json:$.span_attributes`,\n EventsTimestamp Array(DateTime64(9)) `json:$.events_timestamp[:]`,\n EventsName Array(LowCardinality(String)) `json:$.events_name[:]`,\n EventsAttributes Array(Map(LowCardinality(String), String)) `json:$.events_attributes[:]`,\n LinksTraceId Array(String) `json:$.links_trace_id[:]`,\n LinksSpanId Array(String) `json:$.links_span_id[:]`,\n LinksTraceState Array(String) `json:$.links_trace_state[:]`,\n LinksAttributes Array(Map(LowCardinality(String), String)) `json:$.links_attributes[:]`,\n SampleRate Float64 `json:$.SampleRate` DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 `json:$.IsEntryPoint` DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0)\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, ServiceName, SpanName, toDateTime(Timestamp)\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 30 DAY\"\n\nINDEXES >\n idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_span_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1\n idx_resource_attr_items arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) TYPE bloom_filter(0.01) GRANULARITY 1" + }, + { + "name": "traces_aggregates_hourly", + "content": "DESCRIPTION >\n Hourly pre-aggregated trace metrics with sampling-weighted state columns. Generalized MV target for timeseries/breakdown/service-overview queries. AggregatingMergeTree.\n\nSCHEMA >\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n\nENGINE \"AggregatingMergeTree\"\nENGINE_PARTITION_KEY \"toDate(Hour)\"\nENGINE_SORTING_KEY \"OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\"\nENGINE_TTL \"toDate(Hour) + INTERVAL 90 DAY\"" + } ] as const export const pipes = [ - { - name: "error_events_by_time_mv", - content: - "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time", - }, - { - name: "error_events_mv", - content: - "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events", - }, - { - name: "error_spans_mv", - content: - "DESCRIPTION >\n Materializes error spans from traces. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment.\n\nNODE error_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_spans", - }, - { - name: "log_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates log attribute keys from logs hourly.\n\nNODE log_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "log_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates log attribute values from logs hourly.\n\nNODE log_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "logs_aggregates_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates logs hourly by service × severity × deployment env. Drop-in for severity-distribution and log-volume queries.\n\nNODE logs_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\n\nTYPE MATERIALIZED\nDATASOURCE logs_aggregates_hourly", - }, - { - name: "metric_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates metric attribute keys from metrics_sum hourly.\n\nNODE metric_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n arrayJoin(mapKeys(Attributes)) AS AttributeKey,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n WHERE Attributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "metric_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates metric attribute values from metrics_sum hourly.\n\nNODE metric_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "metric_catalog_exp_histogram_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct exponential histogram metrics into metric_catalog.\n\nNODE metric_catalog_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'exponential_histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_exponential_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "metric_catalog_gauge_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct gauge metrics into metric_catalog.\n\nNODE metric_catalog_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "metric_catalog_histogram_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct histogram metrics into metric_catalog.\n\nNODE metric_catalog_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "metric_catalog_sum_mv", - content: - "DESCRIPTION >\n Hourly rollup of distinct sum metrics into metric_catalog.\n\nNODE metric_catalog_sum_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog", - }, - { - name: "service_external_edges_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates Client/Producer spans without db.system.name into hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab.\n\nNODE service_external_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_external_edges_hourly", - }, - { - name: "service_map_children_mv", - content: - "DESCRIPTION >\n Populates service_map_children with Server/Consumer spans that have a parent for efficient JOIN lookups.\n\nNODE service_map_children_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_map_children", - }, - { - name: "service_map_db_edges_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates Client/Producer spans with db.system.name into hourly service-to-database edge buckets for fast service map db-node queries.\n\nNODE service_map_db_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_edges_hourly", - }, - { - name: "service_map_db_query_shapes_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates Client/Producer DB spans into hourly query-shape buckets (normalized fingerprint + label + sample-weighted t-digest) for the service map's database detail panel.\n\nNODE service_map_db_query_shapes_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n )))\n ) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n ), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv, QueryKey\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_query_shapes_hourly", - }, - { - name: "service_map_spans_mv", - content: - "DESCRIPTION >\n Materialized view projecting trace spans needed for service dependency map. Extracts deployment.environment from Map columns at write time.\n\nNODE service_map_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')\n\nTYPE MATERIALIZED\nDATASOURCE service_map_spans", - }, - { - name: "service_overview_spans_mv", - content: - "DESCRIPTION >\n Materialized view projecting service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes at write time.\n\nNODE service_overview_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_spans", - }, - { - name: "service_platforms_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly", - }, - { - name: "service_usage_logs_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate log usage statistics per service per hour\n\nNODE service_usage_logs_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_exp_histogram_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate exponential histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_gauge_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate gauge metric usage statistics per service per hour\n\nNODE service_usage_metrics_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_histogram_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_metrics_sum_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate sum metric usage statistics per service per hour\n\nNODE service_usage_metrics_sum_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "service_usage_traces_mv", - content: - "DESCRIPTION >\n Materialized view to aggregate trace/span usage statistics per service per hour\n\nNODE service_usage_traces_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage", - }, - { - name: "span_metrics_calls_hourly_mv", - content: - "DESCRIPTION >\n Hourly per-series argMax(value) rollup of the span-metrics calls counter into span_metrics_calls_hourly.\n\nNODE span_metrics_calls_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\n\nTYPE MATERIALIZED\nDATASOURCE span_metrics_calls_hourly", - }, - { - name: "trace_detail_spans_mv", - content: - "DESCRIPTION >\n Populates trace_detail_spans with all spans re-sorted by TraceId for fast detail lookups\n\nNODE trace_detail_spans_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes,\n EventsTimestamp,\n EventsName,\n EventsAttributes\n FROM traces\n\nTYPE MATERIALIZED\nDATASOURCE trace_detail_spans", - }, - { - name: "trace_list_mv_mv", - content: - "DESCRIPTION >\n Populates trace_list_mv from root spans with pre-extracted HTTP attributes and normalized span names.\n\nNODE trace_list_mv_node\nSQL >\n SELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE trace_list_mv", - }, - { - name: "trace_resource_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates resource attribute keys from traces hourly.\n\nNODE trace_resource_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE ResourceAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "trace_resource_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates resource attribute values from traces hourly.\n\nNODE trace_resource_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "trace_span_attribute_keys_mv", - content: - "DESCRIPTION >\n Aggregates span attribute keys from traces hourly.\n\nNODE trace_span_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE SpanAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly", - }, - { - name: "trace_span_attribute_values_mv", - content: - "DESCRIPTION >\n Aggregates span attribute values from traces hourly.\n\nNODE trace_span_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly", - }, - { - name: "traces_aggregates_hourly_mv", - content: - "DESCRIPTION >\n Pre-aggregates spans hourly with sample-weighted state columns (count, duration sum, t-digest quantiles, error count). Sample-correct from day one via SampleRate materialized column on traces.\n\nNODE traces_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE traces_aggregates_hourly", - }, + { + "name": "error_events_by_time_mv", + "content": "DESCRIPTION >\n Time-ordered copy of error_events_mv's projection, written to error_events_by_time (sorted by OrgId, Timestamp, FingerprintHash) for recent-window error scans.\n\nNODE error_events_by_time_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events_by_time" + }, + { + "name": "error_events_mv", + "content": "DESCRIPTION >\n Materializes per-occurrence error events from traces. Unwraps the first OTel exception event and computes a cityHash64 FingerprintHash for issue grouping.\n\nNODE error_events_mv_node\nSQL >\n WITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n arraySlice(\n arrayFilter(\n line -> match(line, ':[0-9]+|line [0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n arrayMap(\n line -> replaceRegexpAll(line, ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection (only consulted when _fpFrames = '')\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- Fold into the existing fallback hash slot. Non-JSON path is unchanged.\n multiIf(\n _fpFrames != '', '',\n _isJsonObj, _jsonSig,\n replaceRegexpAll(substring(StatusMessage, 1, 200), '[0-9a-fA-F]{8,}|[0-9]+', '#')\n ) AS _msgFallback,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgFallback) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_events" + }, + { + "name": "error_spans_mv", + "content": "DESCRIPTION >\n Materializes error spans from traces. Pre-filters to StatusCode='Error' and pre-extracts deployment.environment.\n\nNODE error_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'\n\nTYPE MATERIALIZED\nDATASOURCE error_spans" + }, + { + "name": "log_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates log attribute keys from logs hourly.\n\nNODE log_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "log_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates log attribute values from logs hourly.\n\nNODE log_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n ARRAY JOIN\n mapKeys(LogAttributes) AS AttributeKey,\n mapValues(LogAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "logs_aggregates_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates logs hourly by service × severity × deployment env. Drop-in for severity-distribution and log-volume queries.\n\nNODE logs_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(TimestampTime) AS Hour,\n ServiceName,\n SeverityText,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS Count,\n sum(length(Body) + 200) AS SizeBytes,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM logs\n GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace\n\nTYPE MATERIALIZED\nDATASOURCE logs_aggregates_hourly" + }, + { + "name": "metric_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates metric attribute keys from metrics_sum hourly.\n\nNODE metric_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n arrayJoin(mapKeys(Attributes)) AS AttributeKey,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n WHERE Attributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "metric_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates metric attribute values from metrics_sum hourly.\n\nNODE metric_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'metric' AS AttributeScope,\n count() AS UsageCount\n FROM metrics_sum\n ARRAY JOIN\n mapKeys(Attributes) AS AttributeKey,\n mapValues(Attributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "metric_catalog_exp_histogram_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct exponential histogram metrics into metric_catalog.\n\nNODE metric_catalog_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'exponential_histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_exponential_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "metric_catalog_gauge_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct gauge metrics into metric_catalog.\n\nNODE metric_catalog_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "metric_catalog_histogram_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct histogram metrics into metric_catalog.\n\nNODE metric_catalog_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "metric_catalog_sum_mv", + "content": "DESCRIPTION >\n Hourly rollup of distinct sum metrics into metric_catalog.\n\nNODE metric_catalog_sum_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName\n\nTYPE MATERIALIZED\nDATASOURCE metric_catalog" + }, + { + "name": "service_external_edges_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates Client/Producer spans without db.system.name into hourly service-to-external-target edges (http / messaging / rpc) for the service-detail Dependencies tab.\n\nNODE service_external_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '',\n if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR SpanAttributes['messaging.destination'] != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_external_edges_hourly" + }, + { + "name": "service_map_children_mv", + "content": "DESCRIPTION >\n Populates service_map_children with Server/Consumer spans that have a parent for efficient JOIN lookups.\n\nNODE service_map_children_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''\n\nTYPE MATERIALIZED\nDATASOURCE service_map_children" + }, + { + "name": "service_map_db_edges_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates Client/Producer spans with db.system.name into hourly service-to-database edge buckets for fast service map db-node queries.\n\nNODE service_map_db_edges_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_edges_hourly" + }, + { + "name": "service_map_db_query_shapes_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates Client/Producer DB spans into hourly query-shape buckets (normalized fingerprint + label + sample-weighted t-digest) for the service map's database detail panel.\n\nNODE service_map_db_query_shapes_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n coalesce(\n nullIf(SpanAttributes['db.query.fingerprint'], ''),\n nullIf(SpanAttributes['db.statement.fingerprint'], ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\\'[^\\']*\\'', '?'), '\\\\bin\\\\s*\\\\([^)]*\\\\)', 'in (?)'), '[0-9]+(\\\\.[0-9]+)?', '?'), '\\\\s+', ' '), '^\\\\s+|\\\\s+$', ''))), ''), ''),\n toString(cityHash64(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n )))\n ) AS QueryKey,\n any(substring(coalesce(\n nullIf(SpanAttributes['db.query.summary'], ''),\n nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''),\n nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\\\s*(\\\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\\\s+\\\\W?([\\\\w.]+)')), ''))), ''), ''),\n nullIf(SpanAttributes['query.context'], ''),\n nullIf(SpanAttributes['db.operation.name'], ''),\n nullIf(SpanAttributes['db.operation'], ''),\n SpanName\n ), 1, 220)) AS QueryLabel,\n any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(SampleRate) AS EstimatedCount,\n sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount,\n sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DeploymentEnv, QueryKey\n\nTYPE MATERIALIZED\nDATASOURCE service_map_db_query_shapes_hourly" + }, + { + "name": "service_map_spans_mv", + "content": "DESCRIPTION >\n Materialized view projecting trace spans needed for service dependency map. Extracts deployment.environment from Map columns at write time.\n\nNODE service_map_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer')\n\nTYPE MATERIALIZED\nDATASOURCE service_map_spans" + }, + { + "name": "service_overview_spans_mv", + "content": "DESCRIPTION >\n Materialized view projecting service entry point spans (Server/Consumer + root) for service overview queries. Pre-extracts deployment attributes from ResourceAttributes at write time.\n\nNODE service_overview_spans_mv_node\nSQL >\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n Duration,\n StatusCode,\n TraceState,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n ResourceAttributes['deployment.commit_sha'] AS CommitSha,\n SampleRate,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE service_overview_spans" + }, + { + "name": "service_platforms_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates per-service hosting-platform resource attributes (k8s.*, cloud.*, faas.*) into hourly buckets for the service map's runtime-icon resolver.\n\nNODE service_platforms_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster,\n max(ResourceAttributes['k8s.pod.name']) AS K8sPodName,\n max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName,\n max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName,\n max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName,\n max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName,\n max(ResourceAttributes['cloud.platform']) AS CloudPlatform,\n max(ResourceAttributes['cloud.provider']) AS CloudProvider,\n max(ResourceAttributes['faas.name']) AS FaasName,\n max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType,\n max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName,\n count() AS SpanCount\n FROM traces\n WHERE ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE service_platforms_hourly" + }, + { + "name": "service_usage_logs_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate log usage statistics per service per hour\n\nNODE service_usage_logs_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(TimestampTime) AS Hour,\n count() AS LogCount,\n sum(length(Body) + 200) AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM logs\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_exp_histogram_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate exponential histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_exp_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n count() AS ExpHistogramMetricCount,\n count() * 300 AS ExpHistogramMetricSizeBytes\n FROM metrics_exponential_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_gauge_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate gauge metric usage statistics per service per hour\n\nNODE service_usage_metrics_gauge_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n count() AS GaugeMetricCount,\n count() * 150 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_gauge\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_histogram_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate histogram metric usage statistics per service per hour\n\nNODE service_usage_metrics_histogram_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n count() AS HistogramMetricCount,\n count() * 250 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_histogram\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_metrics_sum_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate sum metric usage statistics per service per hour\n\nNODE service_usage_metrics_sum_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n 0 AS TraceCount,\n 0 AS TraceSizeBytes,\n count() AS SumMetricCount,\n count() * 150 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM metrics_sum\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "service_usage_traces_mv", + "content": "DESCRIPTION >\n Materialized view to aggregate trace/span usage statistics per service per hour\n\nNODE service_usage_traces_mv_node\nSQL >\n SELECT\n OrgId,\n ServiceName,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n 0 AS LogCount,\n 0 AS LogSizeBytes,\n count() AS TraceCount,\n sum(length(SpanName) + 300) AS TraceSizeBytes,\n 0 AS SumMetricCount,\n 0 AS SumMetricSizeBytes,\n 0 AS GaugeMetricCount,\n 0 AS GaugeMetricSizeBytes,\n 0 AS HistogramMetricCount,\n 0 AS HistogramMetricSizeBytes,\n 0 AS ExpHistogramMetricCount,\n 0 AS ExpHistogramMetricSizeBytes\n FROM traces\n GROUP BY OrgId, ServiceName, Hour\n\nTYPE MATERIALIZED\nDATASOURCE service_usage" + }, + { + "name": "span_metrics_calls_hourly_mv", + "content": "DESCRIPTION >\n Hourly per-series argMax(value) rollup of the span-metrics calls counter into span_metrics_calls_hourly.\n\nNODE span_metrics_calls_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n ServiceName,\n MetricName,\n Attributes['span.kind'] AS SpanKind,\n cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint,\n cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint,\n StartTimeUnix,\n argMaxState(Value, TimeUnix) AS LastValue\n FROM metrics_sum\n WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic\n GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix\n\nTYPE MATERIALIZED\nDATASOURCE span_metrics_calls_hourly" + }, + { + "name": "trace_detail_spans_mv", + "content": "DESCRIPTION >\n Populates trace_detail_spans with all spans re-sorted by TraceId for fast detail lookups\n\nNODE trace_detail_spans_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n SpanName,\n SpanKind,\n ServiceName,\n Duration,\n StatusCode,\n StatusMessage,\n SpanAttributes,\n ResourceAttributes,\n EventsTimestamp,\n EventsName,\n EventsAttributes\n FROM traces\n\nTYPE MATERIALIZED\nDATASOURCE trace_detail_spans" + }, + { + "name": "trace_list_mv_mv", + "content": "DESCRIPTION >\n Populates trace_list_mv from root spans with pre-extracted HTTP attributes and normalized span names.\n\nNODE trace_list_mv_node\nSQL >\n SELECT\n OrgId,\n TraceId,\n toDateTime(Timestamp) AS Timestamp,\n ServiceName,\n if(\n (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS'))\n AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''),\n concat(\n if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName),\n ' ',\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])\n ),\n SpanName\n ) AS SpanName,\n SpanKind,\n Duration,\n StatusCode,\n if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod,\n if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute,\n if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n toUInt8(\n StatusCode = 'Error'\n OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500)\n OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500)\n ) AS HasError,\n TraceState,\n ResourceAttributes['service.namespace'] AS ServiceNamespace\n FROM traces\n WHERE ParentSpanId = ''\n\nTYPE MATERIALIZED\nDATASOURCE trace_list_mv" + }, + { + "name": "trace_resource_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates resource attribute keys from traces hourly.\n\nNODE trace_resource_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE ResourceAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "trace_resource_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates resource attribute values from traces hourly.\n\nNODE trace_resource_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'resource' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(ResourceAttributes) AS AttributeKey,\n mapValues(ResourceAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "trace_span_attribute_keys_mv", + "content": "DESCRIPTION >\n Aggregates span attribute keys from traces hourly.\n\nNODE trace_span_attribute_keys_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n WHERE SpanAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_keys_hourly" + }, + { + "name": "trace_span_attribute_values_mv", + "content": "DESCRIPTION >\n Aggregates span attribute values from traces hourly.\n\nNODE trace_span_attribute_values_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n AttributeKey,\n AttributeValue,\n 'span' AS AttributeScope,\n count() AS UsageCount\n FROM traces\n ARRAY JOIN\n mapKeys(SpanAttributes) AS AttributeKey,\n mapValues(SpanAttributes) AS AttributeValue\n WHERE AttributeValue != ''\n GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope\n\nTYPE MATERIALIZED\nDATASOURCE attribute_values_hourly" + }, + { + "name": "traces_aggregates_hourly_mv", + "content": "DESCRIPTION >\n Pre-aggregates spans hourly with sample-weighted state columns (count, duration sum, t-digest quantiles, error count). Sample-correct from day one via SampleRate materialized column on traces.\n\nNODE traces_aggregates_hourly_mv_node\nSQL >\n SELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n SpanName,\n SpanKind,\n StatusCode,\n IsEntryPoint,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n sum(SampleRate) AS WeightedCount,\n sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum,\n sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount,\n quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles,\n min(Duration) AS DurationMin,\n max(Duration) AS DurationMax\n FROM traces\n GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv\n\nTYPE MATERIALIZED\nDATASOURCE traces_aggregates_hourly" + } ] as const export const tinybirdProjectManifest = { - projectRevision, - datasources, - pipes, + projectRevision, + datasources, + pipes, } as const diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index ca9e6ecb..fe252941 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -1,4 +1,5 @@ import { defineDatasource, t, engine, column, type InferRow } from "@tinybirdco/sdk" +import { attrItemsExpr, BODY_TOKEN_INDEX_EXPR } from "./index-exprs" /** * OpenTelemetry logs datasource @@ -48,6 +49,43 @@ export const logs = defineDatasource("logs", { type: "bloom_filter(0.01)", granularity: 1, }, + // Full-text token index on the log body. `Body ILIKE '%term%'` otherwise + // scans whole partitions (guard-railed by `LOGS_BODY_SEARCH_SETTINGS`); + // the query engine AND-s a `hasToken(lower(Body), )` pre-filter for + // interior tokens so this index prunes granules. tokenbf params match + // ClickStack: 32 KB filter, 3 hashes, seed 0. + { + name: "idx_body_tokens", + expr: BODY_TOKEN_INDEX_EXPR, + type: "tokenbf_v1(32768, 3, 0)", + granularity: 1, + }, + // Attribute-map blooms, mirroring `traces`. Enable granule skipping for + // `LogAttributes['k'] = 'v'` / key-existence filters on the logs table. + { + name: "idx_log_attr_keys", + expr: "mapKeys(LogAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_log_attr_vals", + expr: "mapValues(LogAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_log_resource_attr_keys", + expr: "mapKeys(ResourceAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_log_resource_attr_vals", + expr: "mapValues(ResourceAttributes)", + type: "bloom_filter(0.01)", + granularity: 1, + }, ], engine: engine.mergeTree({ partitionKey: "toDate(TimestampTime)", @@ -199,6 +237,23 @@ export const traces = defineDatasource("traces", { type: "bloom_filter(0.01)", granularity: 1, }, + // ClickStack "Items" indexes: a bloom over concatenated `key=value` strings + // so an equality filter `SpanAttributes['k'] = 'v'` prunes granules via + // `has(, 'k=v')`. The mapKeys/mapValues blooms above only skip on + // key OR value independently; the items index skips on the exact pair. + // The `expr` MUST match `attrItemsExpr(...)` used by the query engine. + { + name: "idx_span_attr_items", + expr: attrItemsExpr("SpanAttributes"), + type: "bloom_filter(0.01)", + granularity: 1, + }, + { + name: "idx_resource_attr_items", + expr: attrItemsExpr("ResourceAttributes"), + type: "bloom_filter(0.01)", + granularity: 1, + }, ], engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", diff --git a/packages/domain/src/tinybird/index-exprs.ts b/packages/domain/src/tinybird/index-exprs.ts new file mode 100644 index 00000000..a3895a57 --- /dev/null +++ b/packages/domain/src/tinybird/index-exprs.ts @@ -0,0 +1,33 @@ +// --------------------------------------------------------------------------- +// Shared skip-index expressions +// +// These SQL expression strings are used in TWO places that MUST stay byte-for- +// byte identical, otherwise the index silently stops applying: +// +// 1. The `expr` of a data-skipping INDEX on a datasource (datasources.ts) and +// the equivalent `ALTER TABLE ... ADD INDEX` in a BYO-ClickHouse migration. +// 2. The query-engine predicate that references the same expression so +// ClickHouse's index analysis recognizes it (e.g. `has(, …)`). +// +// Keep the single source here and import it on both sides. +// --------------------------------------------------------------------------- + +/** + * ClickStack "Items" expression: concatenates each map entry into a single + * `key=value` string so an equality filter `Map['k'] = 'v'` can be pruned by a + * `bloom_filter` skip index via `has(, 'k=v')`. + * + * `mapKeys`/`mapValues` preserve element order, so `arrayMap` pairs them + * correctly. Applied to `SpanAttributes` / `ResourceAttributes` on `traces`. + */ +export function attrItemsExpr(mapName: string): string { + return `arrayMap((k, v) -> concat(k, '=', v), mapKeys(${mapName}), mapValues(${mapName}))` +} + +/** + * Body full-text token index expression. The index is on `lower(Body)` so a + * case-insensitive `hasToken(lower(Body), '')` pre-filter can + * prune granules ahead of the substring `ILIKE` (see the query engine's + * `body-search.ts`). + */ +export const BODY_TOKEN_INDEX_EXPR = "lower(Body)" diff --git a/packages/domain/src/tinybird/index.ts b/packages/domain/src/tinybird/index.ts index 85a5e176..ee0aa33a 100644 --- a/packages/domain/src/tinybird/index.ts +++ b/packages/domain/src/tinybird/index.ts @@ -29,5 +29,8 @@ export * from "./materializations" // Export shared DB query-shape SQL fragments (label/key derivation) export * from "./db-query-shape-sql" +// Export shared skip-index expressions (kept in sync with query-engine predicates) +export * from "./index-exprs" + // Export TTL override helpers for BYO Tinybird raw retention export * from "./ttl-override" diff --git a/packages/query-engine/src/ch/queries/body-search.test.ts b/packages/query-engine/src/ch/queries/body-search.test.ts new file mode 100644 index 00000000..6c234110 --- /dev/null +++ b/packages/query-engine/src/ch/queries/body-search.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest" +import { compileCH, from } from "@maple-dev/clickhouse-builder" +import { Logs } from "../tables" +import { bodySearchConditions, extractSafeSearchTokens } from "./body-search" + +// --------------------------------------------------------------------------- +// extractSafeSearchTokens — only interior, separator-bounded ASCII tokens +// --------------------------------------------------------------------------- + +describe("extractSafeSearchTokens", () => { + it.each([ + // A single word is one edge-to-edge run — no interior boundary, so unsafe + // (it could be a prefix/suffix of a longer token in Body). + ["exception", []], + ["timeout", []], + // Interior tokens are bounded on both sides by separators within the term. + ["error: user 42 timeout", ["user", "42"]], + ["GET /api/users failed", ["api", "users"]], + // Lowercased to match the `lower(Body)` index. + ["a FOO b", ["foo"]], + // Underscore is not treated as a separator (tokenizer-version-safe). + ["foo_bar baz qux", ["baz"]], + // Non-ASCII disables the pre-filter entirely (JS/CH lower() divergence). + ["café con leche", []], + // Dedup: the second "user" is dropped; edge tokens x/z excluded, y kept. + ["x user y user z", ["user", "y"]], + // Empty / whitespace. + ["", []], + [" ", []], + ])("extractSafeSearchTokens(%j) = %j", (term, expected) => { + expect(extractSafeSearchTokens(term)).toEqual(expected) + }) + + it("caps at 4 tokens", () => { + expect(extractSafeSearchTokens("a b c d e f g h").length).toBe(4) + }) +}) + +// --------------------------------------------------------------------------- +// bodySearchConditions — compiled SQL shape +// --------------------------------------------------------------------------- + +function whereSql(term: string): string { + const q = from(Logs) + .select(($) => ({ body: $.Body })) + .where(($) => bodySearchConditions($.Body, term)) + .format("JSON") + return compileCH(q, {}).sql +} + +describe("bodySearchConditions", () => { + it("emits hasToken pre-filters plus the substring ILIKE for a phrase", () => { + const sql = whereSql("error: user 42 timeout") + expect(sql).toContain("hasToken(lower(Body), 'user')") + expect(sql).toContain("hasToken(lower(Body), '42')") + expect(sql).toContain("Body ILIKE '%error: user 42 timeout%'") + }) + + it("degrades to ILIKE only for a single word (no safe token)", () => { + const sql = whereSql("exception") + expect(sql).not.toContain("hasToken") + expect(sql).toContain("Body ILIKE '%exception%'") + }) +}) diff --git a/packages/query-engine/src/ch/queries/body-search.ts b/packages/query-engine/src/ch/queries/body-search.ts new file mode 100644 index 00000000..3bc4b718 --- /dev/null +++ b/packages/query-engine/src/ch/queries/body-search.ts @@ -0,0 +1,80 @@ +// --------------------------------------------------------------------------- +// Log body search - token pre-filter + substring predicate +// +// `Body ILIKE '%term%'` is substring-matching and scans whole partitions. The +// `logs.idx_body_tokens` skip index (`tokenbf_v1` over `lower(Body)`) can prune +// granules, but only via whole-token `hasToken(...)` predicates. Whole-token +// semantics are NOT the same as substring semantics, so we can only AND a +// `hasToken` pre-filter for tokens that are *guaranteed* to appear as whole +// tokens whenever the substring matches - otherwise we would silently drop rows. +// +// The final `ILIKE` predicate always stays, so the user-visible result set is +// byte-for-byte unchanged; the token predicates are a pure granule-pruning +// accelerator that the query planner is free to satisfy from the index. +// --------------------------------------------------------------------------- + +import * as CH from "@maple-dev/clickhouse-builder/expr" + +/** At most this many token pre-filters (diminishing pruning, longer SQL). */ +const MAX_TOKENS = 4 + +const NON_ASCII = /[^\x00-\x7F]/ +const ALNUM_RUN = /[A-Za-z0-9]+/g + +/** + * Tokens of `term` that are safe to AND as a `hasToken(lower(Body), token)` + * pre-filter without changing the substring semantics of `Body ILIKE '%term%'`. + * + * Soundness: if `term` occurs as a substring of `Body`, an alphanumeric run of + * `term` is guaranteed to be a *whole* ClickHouse token of `Body` only when it + * is bounded on BOTH sides - within `term` itself - by a definite token + * separator. An edge run (touching the start/end of `term`) could be extended + * inside `Body` (`"conn"` inside `"connection"`), so it is unsafe. + * + * Conservative choices (each only ever drops an optimization, never a result): + * - ASCII-only: JS `String.toLowerCase()` and ClickHouse `lower()` agree on + * case folding for ASCII; outside ASCII they diverge, so bail entirely. + * - `_` is NOT treated as a separator. ClickHouse's tokenizer treatment of + * underscore is version-dependent; requiring boundaries in `[^A-Za-z0-9_]` + * keeps us correct whether or not `_` splits tokens. + */ +export function extractSafeSearchTokens(term: string): string[] { + if (NON_ASCII.test(term)) return [] + + const tokens: string[] = [] + const seen = new Set() + ALNUM_RUN.lastIndex = 0 + let match: RegExpExecArray | null + while ((match = ALNUM_RUN.exec(term)) !== null) { + const start = match.index + const end = start + match[0].length + // A maximal `[A-Za-z0-9]+` run's neighbours (if present) are non- + // alphanumeric. Require both neighbours to exist and to be a definite + // separator - i.e. present and not `_`. A missing neighbour = term edge. + const leftBounded = start > 0 && term[start - 1] !== "_" + const rightBounded = end < term.length && term[end] !== "_" + if (!leftBounded || !rightBounded) continue + + const token = match[0].toLowerCase() + if (seen.has(token)) continue + seen.add(token) + tokens.push(token) + if (tokens.length >= MAX_TOKENS) break + } + return tokens +} + +/** + * Conditions for a log body search: safe `hasToken` pre-filters (index-prunable) + * followed by the substring `ILIKE` that fixes the exact semantics. Spread into + * a query's `where` array. When no safe token exists (e.g. a single word) this + * degrades to just the `ILIKE`, i.e. today's behavior. + */ +export function bodySearchConditions(body: CH.Expr, term: string): CH.Condition[] { + const loweredBody = CH.lower_(body) + const conditions: CH.Condition[] = extractSafeSearchTokens(term).map((token) => + CH.hasToken(loweredBody, CH.lit(token)), + ) + conditions.push(body.ilike(`%${term}%`)) + return conditions +} diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index 797b80f4..2b7b7872 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -12,6 +12,7 @@ import type { ColumnDefs } from "@maple-dev/clickhouse-builder/types" import * as T from "@maple-dev/clickhouse-builder/types" import { unionAll, type CHUnionQuery } from "@maple-dev/clickhouse-builder" import { Logs, LogsAggregatesHourly } from "../tables" +import { bodySearchConditions } from "./body-search" import { finalizeTimeseries } from "./series-cap" // --------------------------------------------------------------------------- @@ -356,7 +357,7 @@ export function logsCountQuery(opts: LogsQueryOpts): CHQuery $.SeverityText.eq(v)), CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), - CH.when(opts.search, (v: string) => $.Body.ilike(`%${v}%`)), + ...(opts.search ? bodySearchConditions($.Body, opts.search) : []), environmentCondition($, opts), namespaceCondition($, opts), ]) @@ -451,7 +452,7 @@ export function logsListQuery(opts: LogsListOpts) { CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), CH.when(opts.spanId, (v: string) => $.SpanId.eq(v)), CH.when(opts.cursor, (v: string) => $.Timestamp.lt(v)), - CH.when(opts.search, (v: string) => $.Body.ilike(`%${v}%`)), + ...(opts.search ? bodySearchConditions($.Body, opts.search) : []), environmentCondition($, opts), namespaceCondition($, opts), ] diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts index 3902475e..4ef36b4b 100644 --- a/packages/query-engine/src/ch/queries/query-helpers.ts +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -149,6 +149,11 @@ type TracesBaseWhereColumns = Pick< export function tracesBaseWhereConditions( $: ColumnAccessor, opts: TracesBaseWhereOpts, + config?: { + /** Emit `idx_*_attr_items` pre-filters for equality filters. Set only when + * reading the raw `traces` table, which carries those indexes. */ + itemsIndex?: boolean + }, ): Array { const mm = opts.matchModes const conditions: Array = [ @@ -215,12 +220,12 @@ export function tracesBaseWhereConditions( } if (opts.attributeFilters) { for (const af of opts.attributeFilters) { - conditions.push(buildAttrFilterCondition(af, "SpanAttributes")) + conditions.push(buildAttrFilterCondition(af, "SpanAttributes", { itemsIndex: config?.itemsIndex })) } } if (opts.resourceAttributeFilters) { for (const rf of opts.resourceAttributeFilters) { - conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes")) + conditions.push(buildAttrFilterCondition(rf, "ResourceAttributes", { itemsIndex: config?.itemsIndex })) } } if (opts.excludedServiceNames?.length) { diff --git a/packages/query-engine/src/ch/queries/traces.test.ts b/packages/query-engine/src/ch/queries/traces.test.ts index 3f8e0516..c5131718 100644 --- a/packages/query-engine/src/ch/queries/traces.test.ts +++ b/packages/query-engine/src/ch/queries/traces.test.ts @@ -128,6 +128,65 @@ describe("tracesListQuery", () => { }) }) +// --------------------------------------------------------------------------- +// tracesListQuery — ClickStack "Items" attribute-index pre-filter +// --------------------------------------------------------------------------- + +const SPAN_ITEMS = "has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes), mapValues(SpanAttributes))" +const RESOURCE_ITEMS = + "has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes))" + +describe("tracesListQuery attribute items pre-filter", () => { + it("adds a has(items,'k=v') pre-filter alongside the exact map equality", () => { + const q = tracesListQuery({ + attributeFilters: [{ key: "db.system", value: "postgres", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain(`${SPAN_ITEMS}, 'db.system=postgres')`) + expect(sql).toContain("SpanAttributes['db.system'] = 'postgres'") + }) + + it("ORs the pre-filter across HTTP semconv alias spellings", () => { + const q = tracesListQuery({ + attributeFilters: [{ key: "http.method", value: "GET", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain(`${SPAN_ITEMS}, 'http.method=GET')`) + expect(sql).toContain(`${SPAN_ITEMS}, 'http.request.method=GET')`) + }) + + it("uses the resource items index for resource attribute equality", () => { + const q = tracesListQuery({ + resourceAttributeFilters: [{ key: "deployment.environment", value: "prod", mode: "equals" }], + }) + const { sql } = compileCH(q, baseParams) + expect(sql).toContain(`${RESOURCE_ITEMS}, 'deployment.environment=prod')`) + }) + + it("does NOT emit the items pre-filter for contains, negated, or empty-value filters", () => { + const contains = compileCH( + tracesListQuery({ attributeFilters: [{ key: "db.system", value: "postgres", mode: "contains" }] }), + baseParams, + ).sql + expect(contains).not.toContain("has(arrayMap") + expect(contains).toContain("positionCaseInsensitive") + + const negated = compileCH( + tracesListQuery({ + attributeFilters: [{ key: "db.system", value: "postgres", mode: "equals", negated: true }], + }), + baseParams, + ).sql + expect(negated).not.toContain("has(arrayMap") + + const emptyValue = compileCH( + tracesListQuery({ attributeFilters: [{ key: "db.system", value: "", mode: "equals" }] }), + baseParams, + ).sql + expect(emptyValue).not.toContain("has(arrayMap") + }) +}) + // --------------------------------------------------------------------------- // tracesRootListQuery // --------------------------------------------------------------------------- @@ -274,4 +333,18 @@ describe("spanSearchQuery", () => { expect(sql).not.toContain("FROM trace_detail_spans") expect(sql).toContain("SpanName = 'GET /users'") }) + + it("emits the items pre-filter on the raw table but not on trace_detail_spans", () => { + const filter = { attributeFilters: [{ key: "db.system", value: "postgres", mode: "equals" as const }] } + + // Raw traces scan (no traceId) → has the items index → pre-filter emitted. + const raw = compileCH(spanSearchQuery(filter), baseParams).sql + expect(raw).toContain("has(arrayMap((k, v) -> concat(k, '=', v), mapKeys(SpanAttributes)") + + // trace_detail_spans (traceId set) lacks the items index and is already + // pruned by its sort key → no pre-filter, just the exact equality. + const detail = compileCH(spanSearchQuery({ ...filter, traceId: "trace_123" }), baseParams).sql + expect(detail).not.toContain("has(arrayMap") + expect(detail).toContain("SpanAttributes['db.system'] = 'postgres'") + }) }) diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index d1954d27..6b737af9 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -244,7 +244,8 @@ function buildWhereConditions( $: ColumnAccessor, opts: TracesQueryOpts, ): Array { - return tracesBaseWhereConditions($, opts) + // Reads the raw `traces` table → its `idx_*_attr_items` blooms can prune. + return tracesBaseWhereConditions($, opts, { itemsIndex: true }) } // --------------------------------------------------------------------------- @@ -691,6 +692,7 @@ function spanSearchFrom( opts: SpanSearchOpts, limit: number, offset: number, + itemsIndex: boolean, ) { const q = from(source) .select(($) => ({ @@ -706,7 +708,7 @@ function spanSearchFrom( timestamp: CH.toString_($.Timestamp), })) .where(($) => [ - ...tracesBaseWhereConditions($, opts), + ...tracesBaseWhereConditions($, opts, { itemsIndex }), CH.when(opts.traceId, (v: string) => $.TraceId.eq(v)), ]) .orderBy(["timestamp", "desc"]) @@ -720,11 +722,14 @@ export function spanSearchQuery(opts: SpanSearchOpts) { const limit = opts.limit ?? 20 const offset = opts.offset ?? 0 + // `trace_detail_spans` (the traceId path) has no `idx_*_attr_items` and is + // already pruned to a single trace by its (OrgId, TraceId, SpanId) sort key, + // so the pre-filter is only worthwhile on the raw `traces` scan. if (opts.traceId) { - return spanSearchFrom(TraceDetailSpans, opts, limit, offset) + return spanSearchFrom(TraceDetailSpans, opts, limit, offset, false) } - return spanSearchFrom(Traces, opts, limit, offset) + return spanSearchFrom(Traces, opts, limit, offset, true) } // --------------------------------------------------------------------------- diff --git a/packages/query-engine/src/traces-shared.ts b/packages/query-engine/src/traces-shared.ts index 8e721e1e..4864483a 100644 --- a/packages/query-engine/src/traces-shared.ts +++ b/packages/query-engine/src/traces-shared.ts @@ -43,6 +43,7 @@ export const TRACE_LIST_MV_RESOURCE_MAP: Record = { // --------------------------------------------------------------------------- import * as CH from "@maple-dev/clickhouse-builder/expr" +import { attrItemsExpr } from "@maple/domain/tinybird/index-exprs" // --------------------------------------------------------------------------- // HTTP semconv coalescing @@ -83,6 +84,23 @@ function anyMapContains(mapExpr: CH.Expr>, keys: readonly return cond } +/** + * ClickStack "Items" pre-filter: `has(, 'k0=v') OR has(, 'k1=v')` + * over the `bloom_filter`-indexed `arrayMap((k, v) -> concat(k, '=', v), …)` + * expression on `traces`. Implied by the exact map equality (`map[ki] = v` means + * the item `ki=v` is present), so AND-ing it never changes the result set — it + * only lets the index skip granules. The `attrItemsExpr` string MUST match the + * datasource index expression byte-for-byte (single source in @maple/domain). + */ +function attrItemsPrefilter(mapName: string, keys: readonly string[], value: string): CH.Condition { + const items = CH.rawExpr>(attrItemsExpr(mapName)) + let cond = CH.has(items, CH.lit(`${keys[0]}=${value}`)) + for (let i = 1; i < keys.length; i++) { + cond = cond.or(CH.has(items, CH.lit(`${keys[i]}=${value}`))) + } + return cond +} + /** * Rewrites an HTTP server span name to the display form used by the UI and by * `trace_list_mv.SpanName`: spanName `"http.server GET"` + route → `"GET /api/users"`. @@ -111,6 +129,16 @@ export function httpDisplaySpanName( export function buildAttrFilterCondition( af: AttributeFilter, mapName: "SpanAttributes" | "ResourceAttributes", + opts?: { + /** + * When true, an equality filter also emits a granule-prunable + * `has(, 'k=v')` pre-filter backed by the `idx_*_attr_items` bloom + * index. Enable only for callers reading the raw `traces` table (which + * carries those indexes); leave off for tables without them (e.g. metrics) + * to avoid the per-row `arrayMap` cost with no index to pay it back. + */ + itemsIndex?: boolean + }, ): CH.Condition { const mapExpr = CH.dynamicColumn>(mapName) // Span attributes renamed across OTel semconv versions match either spelling, @@ -139,7 +167,14 @@ export function buildAttrFilterCondition( return CH.toFloat64OrZero(colExpr).lte(Number(value)) } // equals (default) - return colExpr.eq(value) + const eq = colExpr.eq(value) + // The items pre-filter can't represent "missing/empty" (an empty value + // has no `k=` entry) and adds no index benefit to a negated predicate, so + // gate it on a non-empty value and the positive branch only. + if (opts?.itemsIndex && value !== "" && !af.negated) { + return attrItemsPrefilter(mapName, keys, value).and(eq) + } + return eq })() return af.negated ? CH.not(positive) : positive