From 02529843f08cf5be233fa83e9db5b69d45a12f26 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 10 Jun 2026 17:22:20 -0700 Subject: [PATCH 1/7] feat(ci): run db migrations from github ci with environment-scoped secrets (#4957) * feat(ci): run db migrations from github ci with environment-scoped secrets * fix(ci): pass environment input via env var instead of shell interpolation * fix(ci): rename migration environments to db-* to avoid collision with Vercel's Production env * improvement(ci): resolve migration db url from prefixed repo secrets, drop github environments * fix(ci): dev migrations use db:push only, matching previous behavior * improvement(ci): reject pooled (pgbouncer) database urls for migrations * Revert "improvement(ci): reject pooled (pgbouncer) database urls for migrations" This reverts commit 3b80d8387be5ff01e9dfe35621b1dbfec8f72fcb. --- .github/workflows/ci.yml | 35 +++++++++++++++++++++++--------- .github/workflows/migrations.yml | 31 ++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1dc73e648f..a019331463f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,10 +46,33 @@ jobs: echo "ℹ️ Not a release commit" fi + # Run database migrations before images are pushed: the ECR push triggers + # CodePipeline, so migrating first guarantees the schema is in place before + # the new app version deploys (replaces the removed ECS migration sidecar) + migrate: + name: Migrate DB + needs: [test-build] + if: >- + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') + uses: ./.github/workflows/migrations.yml + with: + environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }} + secrets: inherit + + # Same ordering for dev (schema push before the dev image lands in ECR) + migrate-dev: + name: Migrate Dev DB + if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + uses: ./.github/workflows/migrations.yml + with: + environment: dev + secrets: inherit + # Dev: build all 3 images for ECR only (no GHCR, no ARM64) build-dev: name: Build Dev ECR - needs: [detect-version] + needs: [detect-version, migrate-dev] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' runs-on: blacksmith-8vcpu-ubuntu-2404 permissions: @@ -108,7 +131,7 @@ jobs: # Main/staging: build AMD64 images and push to ECR + GHCR build-amd64: name: Build AMD64 - needs: [test-build, detect-version] + needs: [test-build, detect-version, migrate] if: >- github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') @@ -318,14 +341,6 @@ jobs: docker manifest push "${IMAGE_BASE}:${VERSION}" fi - # Run database migrations for dev - migrate-dev: - name: Migrate Dev DB - needs: [build-dev] - if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - uses: ./.github/workflows/migrations.yml - secrets: inherit - # Check if docs changed check-docs-changes: name: Check Docs Changes diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 03f85553aa6..240972f9a03 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -2,7 +2,21 @@ name: Database Migrations on: workflow_call: + inputs: + environment: + description: Target environment (production, staging, or dev) + required: true + type: string workflow_dispatch: + inputs: + environment: + description: Target environment + required: true + type: choice + options: + - production + - staging + - dev permissions: contents: read @@ -35,15 +49,24 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + # The expression maps the explicit environment input to exactly one repo + # secret, so the job never holds another environment's database URL. An + # unknown environment resolves to empty and the guard below fails the job. - name: Apply database schema changes working-directory: ./packages/db env: - DATABASE_URL: ${{ github.ref == 'refs/heads/main' && secrets.DATABASE_URL || github.ref == 'refs/heads/dev' && secrets.DEV_DATABASE_URL || secrets.STAGING_DATABASE_URL }} + DATABASE_URL: ${{ inputs.environment == 'production' && secrets.DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_DATABASE_URL || inputs.environment == 'dev' && secrets.DEV_DATABASE_URL || '' }} + ENVIRONMENT: ${{ inputs.environment }} run: | - if [ "${{ github.ref }}" = "refs/heads/dev" ]; then - echo "Dev environment detected — pushing schema with drizzle-kit (db:push)" + if [ -z "$DATABASE_URL" ]; then + echo "ERROR: no database URL secret resolved for environment '${ENVIRONMENT}'" >&2 + exit 1 + fi + + if [ "${ENVIRONMENT}" = "dev" ]; then + echo "Dev environment — pushing schema directly (db:push)" bun run db:push --force else echo "Applying versioned migrations (db:migrate)" bun run ./scripts/migrate.ts - fi \ No newline at end of file + fi From 6e5cce6c0199adc430101109792bf8fde14dbc8a Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 10 Jun 2026 17:54:26 -0700 Subject: [PATCH 2/7] fix(table): translate column name-keyed wire data for workflow tool calls on internal row routes (#4958) --- .../api/table/[tableId]/rows/[rowId]/route.ts | 12 +- .../api/table/[tableId]/rows/route.test.ts | 220 ++++++++++++++++++ .../sim/app/api/table/[tableId]/rows/route.ts | 41 ++-- .../api/table/[tableId]/rows/upsert/route.ts | 9 +- apps/sim/app/api/table/row-wire.ts | 46 ++++ 5 files changed, 306 insertions(+), 22 deletions(-) create mode 100644 apps/sim/app/api/table/[tableId]/rows/route.test.ts create mode 100644 apps/sim/app/api/table/row-wire.ts diff --git a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts index 13a0762c68b..18486c370f6 100644 --- a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts @@ -13,8 +13,9 @@ import { isZodError, parseRequest, validationErrorResponse } from '@/lib/api/ser import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { RowData } from '@/lib/table' +import type { RowData, TableSchema } from '@/lib/table' import { deleteRow, updateRow } from '@/lib/table' +import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableRowAPI') @@ -72,12 +73,14 @@ export const GET = withRouteHandler(async (request: NextRequest, { params }: Row logger.info(`[${requestId}] Retrieved row ${rowId} from table ${tableId}`) + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) + return NextResponse.json({ success: true, data: { row: { id: row.id, - data: row.data, + data: wire.dataOut(row.data as RowData), position: row.position, createdAt: row.createdAt instanceof Date ? row.createdAt.toISOString() : String(row.createdAt), @@ -123,11 +126,12 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) const updatedRow = await updateRow( { tableId, rowId, - data: validated.data as RowData, + data: wire.dataIn(validated.data as RowData), workspaceId: validated.workspaceId, actorUserId: authResult.userId, }, @@ -148,7 +152,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest, context: RowR data: { row: { id: updatedRow.id, - data: updatedRow.data, + data: wire.dataOut(updatedRow.data), position: updatedRow.position, createdAt: updatedRow.createdAt instanceof Date diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts new file mode 100644 index 00000000000..23a12376e03 --- /dev/null +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -0,0 +1,220 @@ +/** + * @vitest-environment node + */ +import { hybridAuthMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition } from '@/lib/table' + +const { mockCheckAccess, mockInsertRow, mockValidateRowData, mockQueryRows } = vi.hoisted(() => ({ + mockCheckAccess: vi.fn(), + mockInsertRow: vi.fn(), + mockValidateRowData: vi.fn(), + mockQueryRows: vi.fn(), +})) + +vi.mock('@/app/api/table/utils', async () => { + const { NextResponse } = await import('next/server') + return { + checkAccess: mockCheckAccess, + accessError: (result: { status: number }) => + NextResponse.json({ error: 'Access denied' }, { status: result.status }), + } +}) + +vi.mock('@/lib/table', async () => { + // Real column-keys translation functions; the row-wire helper under test + // imports them from this barrel. + const columnKeys = await import('@/lib/table/column-keys') + return { + ...columnKeys, + insertRow: mockInsertRow, + batchInsertRows: vi.fn(), + batchUpdateRows: vi.fn(), + deleteRowsByFilter: vi.fn(), + deleteRowsByIds: vi.fn(), + updateRowsByFilter: vi.fn(), + validateBatchRows: vi.fn(), + validateRowData: mockValidateRowData, + validateRowSize: vi.fn(() => ({ valid: true })), + } +}) + +vi.mock('@/lib/table/service', () => ({ + queryRows: mockQueryRows, +})) + +vi.mock('@/lib/table/sql', () => ({ + TableQueryValidationError: class TableQueryValidationError extends Error {}, +})) + +import { GET, POST } from '@/app/api/table/[tableId]/rows/route' + +function buildTable(): TableDefinition { + return { + id: 'tbl_1', + name: 'People', + description: null, + schema: { + columns: [ + { id: 'col_aaa', name: 'Name', type: 'string' }, + { id: 'col_bbb', name: 'Age', type: 'number' }, + ], + }, + metadata: null, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + archivedAt: null, + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + } +} + +function authAs(authType: 'session' | 'internal_jwt') { + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType, + }) +} + +function callPost(body: Record) { + const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +function callGet(query: Record) { + const params = new URLSearchParams(query) + const req = new NextRequest(`http://localhost:3000/api/table/tbl_1/rows?${params}`, { + method: 'GET', + }) + return GET(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) +} + +describe('POST /api/table/[tableId]/rows', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockValidateRowData.mockResolvedValue({ valid: true }) + mockInsertRow.mockResolvedValue({ + id: 'row_1', + data: { col_aaa: 'Ada', col_bbb: 36 }, + position: 1, + orderKey: 'a0', + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + }) + }) + + it('translates name-keyed data to column ids for internal-JWT (workflow tool) callers', async () => { + authAs('internal_jwt') + + const res = await callPost({ + workspaceId: 'workspace-1', + data: { Name: 'Ada', Age: 36 }, + }) + + expect(res.status).toBe(200) + expect(mockValidateRowData).toHaveBeenCalledWith( + expect.objectContaining({ rowData: { col_aaa: 'Ada', col_bbb: 36 } }) + ) + expect(mockInsertRow).toHaveBeenCalledWith( + expect.objectContaining({ data: { col_aaa: 'Ada', col_bbb: 36 } }), + expect.anything(), + expect.any(String) + ) + + const body = await res.json() + expect(body.data.row.data).toEqual({ Name: 'Ada', Age: 36 }) + }) + + it('passes id-keyed data through untouched for session (UI) callers', async () => { + authAs('session') + + const res = await callPost({ + workspaceId: 'workspace-1', + data: { col_aaa: 'Ada', col_bbb: 36 }, + }) + + expect(res.status).toBe(200) + expect(mockInsertRow).toHaveBeenCalledWith( + expect.objectContaining({ data: { col_aaa: 'Ada', col_bbb: 36 } }), + expect.anything(), + expect.any(String) + ) + + const body = await res.json() + expect(body.data.row.data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) + }) +}) + +describe('GET /api/table/[tableId]/rows', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckAccess.mockResolvedValue({ ok: true, table: buildTable() }) + mockQueryRows.mockResolvedValue({ + rows: [ + { + id: 'row_1', + data: { col_aaa: 'Ada', col_bbb: 36 }, + position: 1, + orderKey: 'a0', + createdAt: new Date('2024-01-01'), + updatedAt: new Date('2024-01-01'), + }, + ], + rowCount: 1, + totalCount: 1, + limit: 100, + offset: 0, + }) + }) + + it('translates name-keyed filter/sort and returns name-keyed rows for internal-JWT callers', async () => { + authAs('internal_jwt') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ Name: { $eq: 'Ada' } }), + sort: JSON.stringify({ Age: 'desc' }), + }) + + expect(res.status).toBe(200) + expect(mockQueryRows).toHaveBeenCalledWith( + expect.objectContaining({ id: 'tbl_1' }), + expect.objectContaining({ + filter: { col_aaa: { $eq: 'Ada' } }, + sort: { col_bbb: 'desc' }, + }), + expect.any(String) + ) + + const body = await res.json() + expect(body.data.rows[0].data).toEqual({ Name: 'Ada', Age: 36 }) + }) + + it('passes id-keyed filter and rows through untouched for session callers', async () => { + authAs('session') + + const res = await callGet({ + workspaceId: 'workspace-1', + filter: JSON.stringify({ col_aaa: { $eq: 'Ada' } }), + }) + + expect(res.status).toBe(200) + expect(mockQueryRows).toHaveBeenCalledWith( + expect.objectContaining({ id: 'tbl_1' }), + expect.objectContaining({ filter: { col_aaa: { $eq: 'Ada' } } }), + expect.any(String) + ) + + const body = await res.json() + expect(body.data.rows[0].data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) + }) +}) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 7b27e463c5d..31708805ad2 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -11,7 +11,7 @@ import { } from '@/lib/api/contracts/tables' import { parseRequest } from '@/lib/api/server' import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { Filter, RowData, Sort, TableSchema } from '@/lib/table' @@ -28,6 +28,7 @@ import { } from '@/lib/table' import { queryRows } from '@/lib/table/service' import { TableQueryValidationError } from '@/lib/table/sql' +import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableRowsAPI') @@ -40,7 +41,8 @@ async function handleBatchInsert( requestId: string, tableId: string, validated: BatchInsertTableRowsBodyInput, - userId: string + userId: string, + authType: AuthTypeValue | undefined ): Promise { const accessResult = await checkAccess(tableId, userId, 'write') if (!accessResult.ok) return accessError(accessResult, requestId, tableId) @@ -54,10 +56,13 @@ async function handleBatchInsert( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + const wire = rowWireTranslators(authType, table.schema as TableSchema) + const rows = (validated.rows as RowData[]).map((row) => wire.dataIn(row)) + // Validate rows before calling service (service also validates, but route-level // validation returns structured HTTP responses) const validation = await validateBatchRows({ - rows: validated.rows as RowData[], + rows, schema: table.schema as TableSchema, tableId, }) @@ -67,7 +72,7 @@ async function handleBatchInsert( const insertedRows = await batchInsertRows( { tableId, - rows: validated.rows as RowData[], + rows, workspaceId: validated.workspaceId, userId, positions: validated.positions, @@ -82,7 +87,7 @@ async function handleBatchInsert( data: { rows: insertedRows.map((r) => ({ id: r.id, - data: r.data, + data: wire.dataOut(r.data), position: r.position, orderKey: r.orderKey ?? undefined, createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : r.createdAt, @@ -129,7 +134,7 @@ export const POST = withRouteHandler( const body = parsed.data.body if ('rows' in body) { - return handleBatchInsert(requestId, tableId, body, authResult.userId) + return handleBatchInsert(requestId, tableId, body, authResult.userId, authResult.authType) } const validated = body @@ -146,7 +151,8 @@ export const POST = withRouteHandler( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const rowData = validated.data as RowData + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) + const rowData = wire.dataIn(validated.data as RowData) // Validate at route level for structured HTTP error responses const validation = await validateRowData({ @@ -176,7 +182,7 @@ export const POST = withRouteHandler( data: { row: { id: row.id, - data: row.data, + data: wire.dataOut(row.data), position: row.position, orderKey: row.orderKey ?? undefined, createdAt: row.createdAt instanceof Date ? row.createdAt.toISOString() : row.createdAt, @@ -264,11 +270,12 @@ export const GET = withRouteHandler( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) const result = await queryRows( table, { - filter: validated.filter as Filter | undefined, - sort: validated.sort, + filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined, + sort: validated.sort ? wire.sortIn(validated.sort) : undefined, limit: validated.limit, offset: validated.offset, includeTotal: validated.includeTotal, @@ -281,7 +288,7 @@ export const GET = withRouteHandler( data: { rows: result.rows.map((r) => ({ id: r.id, - data: r.data, + data: wire.dataOut(r.data), executions: r.executions, position: r.position, orderKey: r.orderKey ?? undefined, @@ -344,7 +351,10 @@ export const PUT = withRouteHandler( return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } - const sizeValidation = validateRowSize(validated.data as RowData) + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) + const patchData = wire.dataIn(validated.data as RowData) + + const sizeValidation = validateRowSize(patchData) if (!sizeValidation.valid) { return NextResponse.json( { error: 'Invalid row data', details: sizeValidation.errors }, @@ -355,8 +365,8 @@ export const PUT = withRouteHandler( const result = await updateRowsByFilter( table, { - filter: validated.filter as Filter, - data: validated.data as RowData, + filter: wire.filterIn(validated.filter as Filter), + data: patchData, limit: validated.limit, actorUserId: authResult.userId, }, @@ -466,10 +476,11 @@ export const DELETE = withRouteHandler( }) } + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) const result = await deleteRowsByFilter( table, { - filter: validated.filter as Filter, + filter: wire.filterIn(validated.filter as Filter), limit: validated.limit, }, requestId diff --git a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts index c34ae686c0b..c8d9184e8c3 100644 --- a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts @@ -7,8 +7,9 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server/validation import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { RowData } from '@/lib/table' +import type { RowData, TableSchema } from '@/lib/table' import { upsertRow } from '@/lib/table' +import { rowWireTranslators } from '@/app/api/table/row-wire' import { accessError, checkAccess } from '@/app/api/table/utils' const logger = createLogger('TableUpsertAPI') @@ -41,11 +42,13 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Upser return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) } + const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) + // conflictTarget passes through untranslated — upsertRow resolves it id-or-name. const upsertResult = await upsertRow( { tableId, workspaceId: validated.workspaceId, - data: validated.data as RowData, + data: wire.dataIn(validated.data as RowData), userId: authResult.userId, conflictTarget: validated.conflictTarget, }, @@ -58,7 +61,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Upser data: { row: { id: upsertResult.row.id, - data: upsertResult.row.data, + data: wire.dataOut(upsertResult.row.data), createdAt: upsertResult.row.createdAt instanceof Date ? upsertResult.row.createdAt.toISOString() diff --git a/apps/sim/app/api/table/row-wire.ts b/apps/sim/app/api/table/row-wire.ts new file mode 100644 index 00000000000..89c4cb93af7 --- /dev/null +++ b/apps/sim/app/api/table/row-wire.ts @@ -0,0 +1,46 @@ +import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' +import type { Filter, RowData, Sort, TableSchema } from '@/lib/table' +import { + buildIdByName, + buildNameById, + filterNamesToIds, + rowDataIdToName, + rowDataNameToId, + sortNamesToIds, +} from '@/lib/table' + +export interface RowWireTranslators { + /** Inbound row data: wire keys → storage column ids. */ + dataIn: (data: RowData) => RowData + /** Outbound row data: storage column ids → wire keys. */ + dataOut: (data: RowData) => RowData + /** Inbound filter: wire field refs → storage column ids. */ + filterIn: (filter: Filter) => Filter + /** Inbound sort: wire field refs → storage column ids. */ + sortIn: (sort: Sort) => Sort +} + +/** + * Wire-keying translators for the internal table row routes, which serve two + * caller kinds: the first-party UI (session auth) speaks stable column ids and + * passes through untouched, while workflow tool executions (internal JWT) speak + * column names — tool enrichment surfaces names to the LLM — and translate + * name↔id at this boundary, mirroring the public v1 routes. + */ +export function rowWireTranslators( + authType: AuthTypeValue | undefined, + schema: TableSchema +): RowWireTranslators { + if (authType !== AuthType.INTERNAL_JWT) { + const identity = (value: T): T => value + return { dataIn: identity, dataOut: identity, filterIn: identity, sortIn: identity } + } + const idByName = buildIdByName(schema) + const nameById = buildNameById(schema) + return { + dataIn: (data) => rowDataNameToId(data, idByName), + dataOut: (data) => rowDataIdToName(data, nameById), + filterIn: (filter) => filterNamesToIds(filter, idByName), + sortIn: (sort) => sortNamesToIds(sort, idByName), + } +} From 284edf0b85a5e2777c3d6c456362e3d9c2642453 Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 10 Jun 2026 18:33:58 -0700 Subject: [PATCH 3/7] improvement(db): opt-in read-replica client + migration runner hardening (#4955) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improvement(db): add opt-in read-replica client and harden migration runner * fix(db): detect wrapped lock-timeout errors, jittered retries, direct migration DSN support * fix(db): audit fixes — unparameterized SET, primary reads for authz scoping, shared error helper * fix(db): pin migration session — disable max_lifetime recycling, guard backend pid across retries * fix(db): gate pid session guard to direct migration connections (pooler pids legitimately vary) * improvement(billing): route display-only usage aggregations to the read replica via executor threading * chore(db): drop call-site justification comments * chore(db): tighten doc comments * ci(migrations): map optional direct-connection DSN secrets * fix(data-drains): add stability window to time cursors so late-visible rows are never skipped * fix(billing): resolve pagination cursor on primary; replica for member ledger display * fix(billing): usage-limits returns cost and limit from one computation --- .github/workflows/migrations.yml | 4 + apps/sim/.env.example | 4 + apps/sim/app/api/billing/route.ts | 6 +- apps/sim/app/api/logs/export/route.ts | 4 +- apps/sim/app/api/logs/stats/route.ts | 6 +- .../[id]/members/[memberId]/route.ts | 7 +- apps/sim/app/api/usage/route.ts | 3 +- .../app/api/users/me/usage-limits/route.ts | 8 +- .../admin/organizations/[id]/billing/route.ts | 4 +- apps/sim/app/api/v1/audit-logs/query.ts | 4 +- .../components/table-grid/table-grid.tsx | 2 +- apps/sim/hooks/use-inline-rename.ts | 2 +- apps/sim/lib/billing/core/billing.ts | 37 ++-- apps/sim/lib/billing/core/organization.ts | 38 ++-- apps/sim/lib/billing/core/usage-log.test.ts | 10 +- apps/sim/lib/billing/core/usage-log.ts | 19 +- apps/sim/lib/billing/core/usage.ts | 93 +++++---- apps/sim/lib/billing/credits/daily-refresh.ts | 24 ++- apps/sim/lib/core/config/env.ts | 1 + .../sim/lib/data-drains/sources/audit-logs.ts | 7 +- .../lib/data-drains/sources/copilot-chats.ts | 15 +- .../lib/data-drains/sources/copilot-runs.ts | 6 +- apps/sim/lib/data-drains/sources/cursor.ts | 11 ++ apps/sim/lib/data-drains/sources/job-logs.ts | 6 +- .../lib/data-drains/sources/workflow-logs.ts | 6 +- apps/sim/lib/logs/list-logs.test.ts | 9 +- apps/sim/lib/logs/list-logs.ts | 6 +- packages/db/.env.example | 8 +- packages/db/db.ts | 24 ++- packages/db/scripts/migrate.ts | 185 +++++++++++++----- packages/testing/src/mocks/database.mock.ts | 30 +-- 31 files changed, 401 insertions(+), 188 deletions(-) diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 240972f9a03..ea5ca453968 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -52,10 +52,14 @@ jobs: # The expression maps the explicit environment input to exactly one repo # secret, so the job never holds another environment's database URL. An # unknown environment resolves to empty and the guard below fails the job. + # MIGRATION_DATABASE_URL is the optional direct (non-pooled) DSN preferred + # by migrate.ts; when the secret is unset it resolves to empty and the + # script falls back to DATABASE_URL. - name: Apply database schema changes working-directory: ./packages/db env: DATABASE_URL: ${{ inputs.environment == 'production' && secrets.DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_DATABASE_URL || inputs.environment == 'dev' && secrets.DEV_DATABASE_URL || '' }} + MIGRATION_DATABASE_URL: ${{ inputs.environment == 'production' && secrets.MIGRATION_DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_MIGRATION_DATABASE_URL || '' }} ENVIRONMENT: ${{ inputs.environment }} run: | if [ -z "$DATABASE_URL" ]; then diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 180e9b56e98..d26ff64e52f 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -1,5 +1,9 @@ # Database (Required) DATABASE_URL="postgresql://postgres:your_password@localhost:5432/simstudio" +# Optional read-replica connection string for offloading heavy read paths +# (logs listing, audit logs, dashboard aggregations). Reads fall back to +# DATABASE_URL when unset. +# DATABASE_REPLICA_URL="" # Authentication (Required unless DISABLE_AUTH=true) BETTER_AUTH_SECRET=your_secret_key # Use `openssl rand -hex 32` to generate, or visit https://www.better-auth.com/docs/installation diff --git a/apps/sim/app/api/billing/route.ts b/apps/sim/app/api/billing/route.ts index 2cc338abd48..364f4d707cc 100644 --- a/apps/sim/app/api/billing/route.ts +++ b/apps/sim/app/api/billing/route.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { db, dbReplica } from '@sim/db' import { member } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, eq } from 'drizzle-orm' @@ -65,7 +65,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } const [billingResult, billingStatus] = await Promise.all([ - getSimplifiedBillingSummary(session.user.id, contextId || undefined), + getSimplifiedBillingSummary(session.user.id, contextId || undefined, dbReplica), getEffectiveBillingStatus(session.user.id), ]) billingData = billingResult @@ -114,7 +114,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } // Get organization-specific billing - const rawBillingData = await getOrganizationBillingData(contextId!) + const rawBillingData = await getOrganizationBillingData(contextId!, dbReplica) if (!rawBillingData) { return NextResponse.json( diff --git a/apps/sim/app/api/logs/export/route.ts b/apps/sim/app/api/logs/export/route.ts index f85816e26a1..560eee71618 100644 --- a/apps/sim/app/api/logs/export/route.ts +++ b/apps/sim/app/api/logs/export/route.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { permissions, workflow, workflowExecutionLogs } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, desc, eq, sql } from 'drizzle-orm' @@ -80,7 +80,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { let offset = 0 try { while (true) { - const rows = await db + const rows = await dbReplica .select(selectColumns) .from(workflowExecutionLogs) .leftJoin(workflow, eq(workflowExecutionLogs.workflowId, workflow.id)) diff --git a/apps/sim/app/api/logs/stats/route.ts b/apps/sim/app/api/logs/stats/route.ts index 930e2e36d39..359a8b7505d 100644 --- a/apps/sim/app/api/logs/stats/route.ts +++ b/apps/sim/app/api/logs/stats/route.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { permissions, workflow, workflowExecutionLogs } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, eq, sql } from 'drizzle-orm' @@ -48,7 +48,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const commonFilters = buildFilterConditions(params, { useSimpleLevelFilter: true }) const whereCondition = commonFilters ? and(workspaceFilter, commonFilters) : workspaceFilter - const boundsQuery = await db + const boundsQuery = await dbReplica .select({ minTime: sql`MIN(${workflowExecutionLogs.startedAt})`, maxTime: sql`MAX(${workflowExecutionLogs.startedAt})`, @@ -83,7 +83,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const segmentMs = Math.max(60000, Math.floor(totalMs / params.segmentCount)) const startTimeIso = startTime.toISOString() - const statsQuery = await db + const statsQuery = await dbReplica .select({ workflowId: sql`COALESCE(${workflowExecutionLogs.workflowId}, 'deleted')`, workflowName: sql`COALESCE(${workflow.name}, 'Deleted Workflow')`, diff --git a/apps/sim/app/api/organizations/[id]/members/[memberId]/route.ts b/apps/sim/app/api/organizations/[id]/members/[memberId]/route.ts index 0cb2dd979b4..f6f3dd68944 100644 --- a/apps/sim/app/api/organizations/[id]/members/[memberId]/route.ts +++ b/apps/sim/app/api/organizations/[id]/members/[memberId]/route.ts @@ -1,5 +1,5 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' +import { db, dbReplica } from '@sim/db' import { member, user, userStats } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, eq } from 'drizzle-orm' @@ -97,7 +97,7 @@ export const GET = withRouteHandler( .where(eq(userStats.userId, memberId)) .limit(1) - const computed = await getUserUsageData(memberId) + const computed = await getUserUsageData(memberId, dbReplica) if (usageData.length > 0) { // currentPeriodCost is only a baseline; add this member's attributed @@ -109,7 +109,8 @@ export const GET = withRouteHandler( organizationId, computed.billingPeriodStart && computed.billingPeriodEnd ? { start: computed.billingPeriodStart, end: computed.billingPeriodEnd } - : null + : null, + dbReplica ) ).get(memberId) ?? 0 memberData = { diff --git a/apps/sim/app/api/usage/route.ts b/apps/sim/app/api/usage/route.ts index 5ee3f0a5401..c8271ace599 100644 --- a/apps/sim/app/api/usage/route.ts +++ b/apps/sim/app/api/usage/route.ts @@ -1,3 +1,4 @@ +import { dbReplica } from '@sim/db' import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { getUsageLimitContract, updateUsageLimitContract } from '@/lib/api/contracts/subscription' @@ -62,7 +63,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Permission denied' }, { status: 403 }) } - const org = await getOrganizationBillingData(organizationId) + const org = await getOrganizationBillingData(organizationId, dbReplica) return NextResponse.json({ success: true, context, diff --git a/apps/sim/app/api/users/me/usage-limits/route.ts b/apps/sim/app/api/users/me/usage-limits/route.ts index 4dfd60b1a0a..8f2b18d024e 100644 --- a/apps/sim/app/api/users/me/usage-limits/route.ts +++ b/apps/sim/app/api/users/me/usage-limits/route.ts @@ -5,7 +5,6 @@ import { usageLimitsRequestSchema } from '@/lib/api/contracts/usage-limits' import { AuthType, checkHybridAuth } from '@/lib/auth/hybrid' import { checkServerSideUsageLimits } from '@/lib/billing' import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' -import { getEffectiveCurrentPeriodCost } from '@/lib/billing/core/usage' import { getUserStorageLimit, getUserStorageUsage } from '@/lib/billing/storage' import { RateLimiter } from '@/lib/core/rate-limiter' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -41,14 +40,15 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ), ]) - const [usageCheck, effectiveCost, storageUsage, storageLimit] = await Promise.all([ + const [usageCheck, storageUsage, storageLimit] = await Promise.all([ checkServerSideUsageLimits(authenticatedUserId), - getEffectiveCurrentPeriodCost(authenticatedUserId), getUserStorageUsage(authenticatedUserId), getUserStorageLimit(authenticatedUserId), ]) - const currentPeriodCost = effectiveCost + // Same computation as `limit` (one source, one tier) — the pair can never + // disagree under replication lag or mixed baseline/ledger tiers. + const currentPeriodCost = usageCheck.currentUsage return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts b/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts index 9dd083a30a2..c19168b7a6f 100644 --- a/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts +++ b/apps/sim/app/api/v1/admin/organizations/[id]/billing/route.ts @@ -15,7 +15,7 @@ * Response: AdminSingleResponse<{ success: true, orgUsageLimit: string | null }> */ -import { db } from '@sim/db' +import { db, dbReplica } from '@sim/db' import { member, organization } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { count, eq } from 'drizzle-orm' @@ -96,7 +96,7 @@ export const GET = withRouteHandler( return singleResponse(data) } - const billingData = await getOrganizationBillingData(organizationId) + const billingData = await getOrganizationBillingData(organizationId, dbReplica) if (!billingData) { return notFoundResponse('Organization or subscription') diff --git a/apps/sim/app/api/v1/audit-logs/query.ts b/apps/sim/app/api/v1/audit-logs/query.ts index 8a4173be863..795c54ecfb4 100644 --- a/apps/sim/app/api/v1/audit-logs/query.ts +++ b/apps/sim/app/api/v1/audit-logs/query.ts @@ -1,5 +1,5 @@ import { AuditResourceType } from '@sim/audit' -import { db } from '@sim/db' +import { db, dbReplica } from '@sim/db' import { auditLog, workspace } from '@sim/db/schema' import type { InferSelectModel } from 'drizzle-orm' import { and, desc, eq, gte, ilike, inArray, isNull, lt, lte, or, type SQL, sql } from 'drizzle-orm' @@ -156,7 +156,7 @@ export async function queryAuditLogs( if (cursorCondition) allConditions.push(cursorCondition) } - const rows = await db + const rows = await dbReplica .select() .from(auditLog) .where(allConditions.length > 0 ? and(...allConditions) : undefined) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 9705169c036..c95c2e66090 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -845,7 +845,7 @@ export function TableGrid({ const oldName = columnsRef.current.find((c) => c.key === columnName)?.name ?? columnName pushUndoRef.current({ type: 'rename-column', oldName, newName, columnId: columnName }) handleColumnRename(columnName, newName) - updateColumnMutation.mutate({ columnName, updates: { name: newName } }) + return updateColumnMutation.mutateAsync({ columnName, updates: { name: newName } }) }, }) diff --git a/apps/sim/hooks/use-inline-rename.ts b/apps/sim/hooks/use-inline-rename.ts index 87c819e1a3b..5f0cba4fabf 100644 --- a/apps/sim/hooks/use-inline-rename.ts +++ b/apps/sim/hooks/use-inline-rename.ts @@ -9,7 +9,7 @@ interface UseInlineRenameProps { * `mutateAsync(...)`) — NOT a fire-and-forget `mutate(...)` — so `isSaving` * spans the in-flight request and a rejection can revive the edit session. */ - onSave: (id: string, newName: string) => void | Promise + onSave: (id: string, newName: string) => undefined | Promise } /** diff --git a/apps/sim/lib/billing/core/billing.ts b/apps/sim/lib/billing/core/billing.ts index 2db6b3ef7c7..aca33c6e3d7 100644 --- a/apps/sim/lib/billing/core/billing.ts +++ b/apps/sim/lib/billing/core/billing.ts @@ -22,6 +22,7 @@ import { isOrgScopedSubscription, } from '@/lib/billing/subscriptions/utils' import { Decimal, toDecimal, toNumber } from '@/lib/billing/utils/decimal' +import type { DbOrTx } from '@/lib/db/types' export { getPlanPricing } @@ -384,7 +385,8 @@ export async function calculateSubscriptionOverage(sub: { */ export async function getSimplifiedBillingSummary( userId: string, - organizationId?: string + organizationId?: string, + executor: DbOrTx = db ): Promise<{ type: 'individual' | 'organization' plan: string @@ -432,7 +434,7 @@ export async function getSimplifiedBillingSummary( organizationId ? getOrganizationSubscription(organizationId) : getHighestPrioritySubscription(userId), - getUserUsageData(userId), + getUserUsageData(userId, executor), ]) const plan = subscription?.plan || 'free' @@ -469,7 +471,9 @@ export async function getSimplifiedBillingSummary( const ledgerUsage = orgBillingPeriod ? await getBillingPeriodUsageCost( { type: 'organization', id: organizationId }, - orgBillingPeriod + orgBillingPeriod, + undefined, + executor ) : 0 // Copilot breakdown = member baselines (copilot + MCP) + the copilot-family @@ -481,7 +485,8 @@ export async function getSimplifiedBillingSummary( ? await getBillingPeriodUsageCost( { type: 'organization', id: organizationId }, orgBillingPeriod, - COPILOT_USAGE_SOURCES + COPILOT_USAGE_SOURCES, + executor ) : 0) let refreshDeduction = 0 @@ -492,15 +497,18 @@ export async function getSimplifiedBillingSummary( organizationId, subscription.periodStart ) - refreshDeduction = await computeDailyRefreshConsumed({ - userIds: pooled.memberIds, - periodStart: subscription.periodStart, - periodEnd: subscription.periodEnd ?? null, - planDollars, - seats: subscription.seats || 1, - userBounds: Object.keys(userBounds).length > 0 ? userBounds : undefined, - billingEntity: { type: 'organization', id: organizationId }, - }) + refreshDeduction = await computeDailyRefreshConsumed( + { + userIds: pooled.memberIds, + periodStart: subscription.periodStart, + periodEnd: subscription.periodEnd ?? null, + planDollars, + seats: subscription.seats || 1, + userBounds: Object.keys(userBounds).length > 0 ? userBounds : undefined, + billingEntity: { type: 'organization', id: organizationId }, + }, + executor + ) } } const effectiveCurrentUsage = Math.max(0, rawCurrentUsage + ledgerUsage - refreshDeduction) @@ -609,7 +617,8 @@ export async function getSimplifiedBillingSummary( totalCopilotCost += await getBillingPeriodUsageCost( copilotEntity, copilotBillingPeriod, - COPILOT_USAGE_SOURCES + COPILOT_USAGE_SOURCES, + executor ) } diff --git a/apps/sim/lib/billing/core/organization.ts b/apps/sim/lib/billing/core/organization.ts index dd3389213f9..f7240550ebf 100644 --- a/apps/sim/lib/billing/core/organization.ts +++ b/apps/sim/lib/billing/core/organization.ts @@ -19,6 +19,7 @@ import { hasUsableSubscriptionStatus, } from '@/lib/billing/subscriptions/utils' import { toDecimal, toNumber } from '@/lib/billing/utils/decimal' +import type { DbOrTx } from '@/lib/db/types' const logger = createLogger('OrganizationBilling') @@ -64,7 +65,8 @@ interface MemberUsageData { */ export async function getOrgMemberLedgerByUser( organizationId: string, - period?: { start: Date; end: Date } | null + period?: { start: Date; end: Date } | null, + executor: DbOrTx = db ): Promise> { let billingPeriod = period ?? null if (period === undefined) { @@ -77,7 +79,9 @@ export async function getOrgMemberLedgerByUser( if (!billingPeriod) return new Map() return getBillingPeriodUsageCostByUser( { type: 'organization', id: organizationId }, - billingPeriod + billingPeriod, + undefined, + executor ) } @@ -85,7 +89,8 @@ export async function getOrgMemberLedgerByUser( * Get comprehensive organization billing and usage data */ export async function getOrganizationBillingData( - organizationId: string + organizationId: string, + executor: DbOrTx = db ): Promise { try { // Get organization info @@ -134,7 +139,7 @@ export async function getOrganizationBillingData( subscription.periodStart && subscription.periodEnd ? { start: subscription.periodStart, end: subscription.periodEnd } : null - const usageByUser = await getOrgMemberLedgerByUser(organizationId, billingPeriod) + const usageByUser = await getOrgMemberLedgerByUser(organizationId, billingPeriod, executor) // Process member data const members: MemberUsageData[] = membersWithUsage.map((memberRecord) => { @@ -168,7 +173,9 @@ export async function getOrganizationBillingData( if (billingPeriod) { totalCurrentUsage += await getBillingPeriodUsageCost( { type: 'organization', id: subscription.referenceId }, - billingPeriod + billingPeriod, + undefined, + executor ) } @@ -180,15 +187,18 @@ export async function getOrganizationBillingData( subscription.referenceId, subscription.periodStart ) - const refreshConsumed = await computeDailyRefreshConsumed({ - userIds: memberIds, - periodStart: subscription.periodStart, - periodEnd: subscription.periodEnd ?? null, - planDollars, - seats: subscription.seats || 1, - userBounds: Object.keys(userBounds).length > 0 ? userBounds : undefined, - billingEntity: { type: 'organization', id: subscription.referenceId }, - }) + const refreshConsumed = await computeDailyRefreshConsumed( + { + userIds: memberIds, + periodStart: subscription.periodStart, + periodEnd: subscription.periodEnd ?? null, + planDollars, + seats: subscription.seats || 1, + userBounds: Object.keys(userBounds).length > 0 ? userBounds : undefined, + billingEntity: { type: 'organization', id: subscription.referenceId }, + }, + executor + ) totalCurrentUsage = Math.max(0, totalCurrentUsage - refreshConsumed) } } diff --git a/apps/sim/lib/billing/core/usage-log.test.ts b/apps/sim/lib/billing/core/usage-log.test.ts index 448cb2eab42..ff2f446c832 100644 --- a/apps/sim/lib/billing/core/usage-log.test.ts +++ b/apps/sim/lib/billing/core/usage-log.test.ts @@ -23,12 +23,10 @@ const { mockUpdate: vi.fn(), })) -vi.mock('@sim/db', () => ({ - db: { - insert: mockInsert, - transaction: mockTransaction, - }, -})) +vi.mock('@sim/db', () => { + const instance = { insert: mockInsert, transaction: mockTransaction } + return { db: instance, dbReplica: instance } +}) vi.mock('@sim/db/schema', () => ({ usageLog: { diff --git a/apps/sim/lib/billing/core/usage-log.ts b/apps/sim/lib/billing/core/usage-log.ts index 5d654723d49..147e0406146 100644 --- a/apps/sim/lib/billing/core/usage-log.ts +++ b/apps/sim/lib/billing/core/usage-log.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto' -import { db } from '@sim/db' +import { db, dbReplica } from '@sim/db' import { usageLog, workspace } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' @@ -169,7 +169,8 @@ async function resolveBillingContext( export async function getBillingPeriodUsageCost( billingEntity: BillingEntity, billingPeriod: { start: Date; end: Date }, - source?: UsageLogSource | UsageLogSource[] + source?: UsageLogSource | UsageLogSource[], + executor: DbOrTx = db ): Promise { const conditions = [ eq(usageLog.billingEntityType, billingEntity.type), @@ -183,7 +184,7 @@ export async function getBillingPeriodUsageCost( ) } - const [row] = await db + const [row] = await executor .select({ cost: sql`COALESCE(SUM(${usageLog.cost}), 0)`, }) @@ -196,7 +197,8 @@ export async function getBillingPeriodUsageCost( export async function getBillingPeriodUsageCostByUser( billingEntity: BillingEntity, billingPeriod: { start: Date; end: Date }, - source?: UsageLogSource | UsageLogSource[] + source?: UsageLogSource | UsageLogSource[], + executor: DbOrTx = db ): Promise> { const conditions = [ eq(usageLog.billingEntityType, billingEntity.type), @@ -210,7 +212,7 @@ export async function getBillingPeriodUsageCostByUser( ) } - const rows = await db + const rows = await executor .select({ userId: usageLog.userId, cost: sql`COALESCE(SUM(${usageLog.cost}), 0)`, @@ -579,6 +581,9 @@ export async function getUserUsageLogs( } if (cursor) { + // Cursor resolution stays on the primary: the page itself reads a + // load-balanced replica, and a laggier sibling replica missing the cursor + // row would silently restart pagination from page 1. const cursorLog = await db .select({ createdAt: usageLog.createdAt }) .from(usageLog) @@ -592,7 +597,7 @@ export async function getUserUsageLogs( } } - const logs = await db + const logs = await dbReplica .select() .from(usageLog) .where(and(...conditions)) @@ -621,7 +626,7 @@ export async function getUserUsageLogs( if (startDate) summaryConditions.push(gte(usageLog.createdAt, startDate)) if (endDate) summaryConditions.push(lte(usageLog.createdAt, endDate)) - const summaryResult = await db + const summaryResult = await dbReplica .select({ source: usageLog.source, totalCost: sql`SUM(${usageLog.cost})`, diff --git a/apps/sim/lib/billing/core/usage.ts b/apps/sim/lib/billing/core/usage.ts index 972a199b0fb..7845312d316 100644 --- a/apps/sim/lib/billing/core/usage.ts +++ b/apps/sim/lib/billing/core/usage.ts @@ -34,6 +34,7 @@ import type { BillingData, UsageData, UsageLimitInfo } from '@/lib/billing/types import { Decimal, toDecimal, toNumber } from '@/lib/billing/utils/decimal' import { isBillingEnabled } from '@/lib/core/config/feature-flags' import { getBaseUrl } from '@/lib/core/utils/urls' +import type { DbOrTx } from '@/lib/db/types' import { sendEmail } from '@/lib/messaging/email/mailer' import { getEmailPreferences } from '@/lib/messaging/email/unsubscribe' @@ -179,7 +180,7 @@ export async function ensureUserStatsExists(userId: string): Promise { /** * Get comprehensive usage data for a user */ -export async function getUserUsageData(userId: string): Promise { +export async function getUserUsageData(userId: string, executor: DbOrTx = db): Promise { try { await ensureUserStatsExists(userId) @@ -203,7 +204,12 @@ export async function getUserUsageData(userId: string): Promise { let currentUsageDecimal = toDecimal(stats.currentPeriodCost) if (!orgScoped) { currentUsageDecimal = currentUsageDecimal.plus( - await getBillingPeriodUsageCost({ type: 'user', id: userId }, billingPeriod) + await getBillingPeriodUsageCost( + { type: 'user', id: userId }, + billingPeriod, + undefined, + executor + ) ) } @@ -242,7 +248,9 @@ export async function getUserUsageData(userId: string): Promise { lastPeriodCost = pooled.lastPeriodCost const ledgerUsage = await getBillingPeriodUsageCost( { type: 'organization', id: subscription.referenceId }, - billingPeriod + billingPeriod, + undefined, + executor ) currentUsage = pooled.currentPeriodCost + ledgerUsage } else { @@ -264,24 +272,30 @@ export async function getUserUsageData(userId: string): Promise { subscription.referenceId, billingPeriodStart ) - dailyRefreshConsumed = await computeDailyRefreshConsumed({ - userIds: orgMemberIds, + dailyRefreshConsumed = await computeDailyRefreshConsumed( + { + userIds: orgMemberIds, + periodStart: billingPeriodStart, + periodEnd: billingPeriodEnd, + planDollars, + seats: subscription.seats || 1, + userBounds: Object.keys(userBounds).length > 0 ? userBounds : undefined, + billingEntity: { type: 'organization', id: subscription.referenceId }, + }, + executor + ) + } + } else { + dailyRefreshConsumed = await computeDailyRefreshConsumed( + { + userIds: [userId], periodStart: billingPeriodStart, periodEnd: billingPeriodEnd, planDollars, - seats: subscription.seats || 1, - userBounds: Object.keys(userBounds).length > 0 ? userBounds : undefined, - billingEntity: { type: 'organization', id: subscription.referenceId }, - }) - } - } else { - dailyRefreshConsumed = await computeDailyRefreshConsumed({ - userIds: [userId], - periodStart: billingPeriodStart, - periodEnd: billingPeriodEnd, - planDollars, - billingEntity: { type: 'user', id: userId }, - }) + billingEntity: { type: 'user', id: userId }, + }, + executor + ) } } } @@ -612,7 +626,10 @@ export async function syncUsageLimitsFromSubscription(userId: string): Promise { +export async function getEffectiveCurrentPeriodCost( + userId: string, + executor: DbOrTx = db +): Promise { const subscription = await getHighestPrioritySubscription(userId) const orgScoped = isOrgScopedSubscription(subscription, userId) @@ -631,7 +648,9 @@ export async function getEffectiveCurrentPeriodCost(userId: string): Promise 0 ? userBounds : undefined, - billingEntity: - orgScoped && subscription - ? { type: 'organization', id: subscription.referenceId } - : { type: 'user', id: userId }, - }) + const refreshConsumed = await computeDailyRefreshConsumed( + { + userIds: refreshUserIds, + periodStart: subscription.periodStart, + periodEnd: subscription.periodEnd ?? null, + planDollars, + seats: subscription.seats || 1, + userBounds: Object.keys(userBounds).length > 0 ? userBounds : undefined, + billingEntity: + orgScoped && subscription + ? { type: 'organization', id: subscription.referenceId } + : { type: 'user', id: userId }, + }, + executor + ) return Math.max(0, rawCost - refreshConsumed) } diff --git a/apps/sim/lib/billing/credits/daily-refresh.ts b/apps/sim/lib/billing/credits/daily-refresh.ts index 4835c091f31..7391c74192a 100644 --- a/apps/sim/lib/billing/credits/daily-refresh.ts +++ b/apps/sim/lib/billing/credits/daily-refresh.ts @@ -16,6 +16,7 @@ import { member, usageLog, userStats } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, eq, gte, inArray, lt, or, sql, sum } from 'drizzle-orm' import { DAILY_REFRESH_RATE } from '@/lib/billing/constants' +import type { DbOrTx } from '@/lib/db/types' const logger = createLogger('DailyRefresh') @@ -41,15 +42,18 @@ export interface PerUserBounds { * * @returns Total dollars of refresh consumed across all days (to subtract from usage) */ -export async function computeDailyRefreshConsumed(params: { - userIds: string[] - periodStart: Date - periodEnd?: Date | null - planDollars: number - seats?: number - userBounds?: Record - billingEntity?: { type: 'user' | 'organization'; id: string } -}): Promise { +export async function computeDailyRefreshConsumed( + params: { + userIds: string[] + periodStart: Date + periodEnd?: Date | null + planDollars: number + seats?: number + userBounds?: Record + billingEntity?: { type: 'user' | 'organization'; id: string } + }, + executor: DbOrTx = db +): Promise { const { userIds, periodStart, @@ -114,7 +118,7 @@ export async function computeDailyRefreshConsumed(params: { if (rowFilters.length === 0) return 0 - const rows = await db + const rows = await executor .select({ dayIndex: sql`FLOOR((EXTRACT(EPOCH FROM ${usageLog.createdAt}) - ${Math.floor(periodStart.getTime() / 1000)}) / 86400)`.as( diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 87af1a75f35..7ba9c92511d 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -19,6 +19,7 @@ export const env = createEnv({ server: { // Core Database & Authentication DATABASE_URL: z.string().url(), // Primary database connection string + DATABASE_REPLICA_URL: z.string().url().optional(), // Read-replica connection string; opt-in reads fall back to the primary when unset BETTER_AUTH_URL: z.string().url(), // Base URL for Better Auth service BETTER_AUTH_SECRET: z.string().min(32), // Secret key for Better Auth JWT signing DISABLE_REGISTRATION: z.boolean().optional(), // Flag to disable new user registration diff --git a/apps/sim/lib/data-drains/sources/audit-logs.ts b/apps/sim/lib/data-drains/sources/audit-logs.ts index fbbbbf77aa1..b94ce50ae8b 100644 --- a/apps/sim/lib/data-drains/sources/audit-logs.ts +++ b/apps/sim/lib/data-drains/sources/audit-logs.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { auditLog } from '@sim/db/schema' import { and, inArray, isNull, or, sql } from 'drizzle-orm' import { @@ -6,6 +6,7 @@ import { encodeTimeCursor, timeCursorOrderBy, timeCursorPredicate, + timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' @@ -35,10 +36,10 @@ async function* pages(input: SourcePageInput): AsyncIterable { while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(auditLog.createdAt, auditLog.id, cursor) - const rows = await db + const rows = await dbReplica .select() .from(auditLog) - .where(and(scopeClause, cursorClause)) + .where(and(scopeClause, timeCursorStabilityBound(auditLog.createdAt), cursorClause)) .orderBy(...timeCursorOrderBy(auditLog.createdAt, auditLog.id)) .limit(input.chunkSize) diff --git a/apps/sim/lib/data-drains/sources/copilot-chats.ts b/apps/sim/lib/data-drains/sources/copilot-chats.ts index 6bde4632dca..34a16096381 100644 --- a/apps/sim/lib/data-drains/sources/copilot-chats.ts +++ b/apps/sim/lib/data-drains/sources/copilot-chats.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { copilotChats, copilotMessages } from '@sim/db/schema' import { and, asc, inArray, isNull, sql } from 'drizzle-orm' import { @@ -6,6 +6,7 @@ import { encodeTimeCursor, timeCursorOrderBy, timeCursorPredicate, + timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' @@ -55,17 +56,23 @@ async function* pages(input: SourcePageInput): AsyncIterable { while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(copilotChats.createdAt, copilotChats.id, cursor) - const metaRows = await db + const metaRows = await dbReplica .select(chatColumns) .from(copilotChats) - .where(and(inArray(copilotChats.workspaceId, workspaceIds), cursorClause)) + .where( + and( + inArray(copilotChats.workspaceId, workspaceIds), + timeCursorStabilityBound(copilotChats.createdAt), + cursorClause + ) + ) .orderBy(...timeCursorOrderBy(copilotChats.createdAt, copilotChats.id)) .limit(input.chunkSize) if (metaRows.length === 0) return const chatIds = metaRows.map((r) => r.id) - const messageRows = await db + const messageRows = await dbReplica .select({ chatId: copilotMessages.chatId, content: copilotMessages.content }) .from(copilotMessages) .where(and(inArray(copilotMessages.chatId, chatIds), isNull(copilotMessages.deletedAt))) diff --git a/apps/sim/lib/data-drains/sources/copilot-runs.ts b/apps/sim/lib/data-drains/sources/copilot-runs.ts index 4b2b0503ae7..698008916e4 100644 --- a/apps/sim/lib/data-drains/sources/copilot-runs.ts +++ b/apps/sim/lib/data-drains/sources/copilot-runs.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { copilotRuns } from '@sim/db/schema' import { and, inArray, isNotNull } from 'drizzle-orm' import { @@ -6,6 +6,7 @@ import { encodeTimeCursor, timeCursorOrderBy, timeCursorPredicate, + timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' @@ -24,13 +25,14 @@ async function* pages(input: SourcePageInput): AsyncIterable { while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(copilotRuns.completedAt, copilotRuns.id, cursor) - const rows = await db + const rows = await dbReplica .select() .from(copilotRuns) .where( and( inArray(copilotRuns.workspaceId, workspaceIds), isNotNull(copilotRuns.completedAt), + timeCursorStabilityBound(copilotRuns.completedAt), cursorClause ) ) diff --git a/apps/sim/lib/data-drains/sources/cursor.ts b/apps/sim/lib/data-drains/sources/cursor.ts index 484e5eb1273..b133a6449d0 100644 --- a/apps/sim/lib/data-drains/sources/cursor.ts +++ b/apps/sim/lib/data-drains/sources/cursor.ts @@ -54,3 +54,14 @@ export function timeCursorPredicate( export function timeCursorOrderBy(timestampCol: PgColumn, idCol: PgColumn): [SQL, SQL] { return [sql`date_trunc('milliseconds', ${timestampCol}) asc`, sql`${idCol} asc`] } + +/** + * Excludes rows newer than a short stability window. Timestamp cursors assume + * rows become visible in timestamp order, but out-of-order commits and replica + * lag can surface an earlier-stamped row after the cursor has advanced past it + * — permanently skipping it. Leaving the freshest rows for the next run bounds + * both. + */ +export function timeCursorStabilityBound(timestampCol: PgColumn): SQL { + return sql`${timestampCol} <= now() - interval '5 minutes'` +} diff --git a/apps/sim/lib/data-drains/sources/job-logs.ts b/apps/sim/lib/data-drains/sources/job-logs.ts index 789118e6e67..dcb2d9c98c5 100644 --- a/apps/sim/lib/data-drains/sources/job-logs.ts +++ b/apps/sim/lib/data-drains/sources/job-logs.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { jobExecutionLogs } from '@sim/db/schema' import { and, inArray, isNotNull } from 'drizzle-orm' import { @@ -6,6 +6,7 @@ import { encodeTimeCursor, timeCursorOrderBy, timeCursorPredicate, + timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' @@ -24,13 +25,14 @@ async function* pages(input: SourcePageInput): AsyncIterable { while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(jobExecutionLogs.endedAt, jobExecutionLogs.id, cursor) - const rows = await db + const rows = await dbReplica .select() .from(jobExecutionLogs) .where( and( inArray(jobExecutionLogs.workspaceId, workspaceIds), isNotNull(jobExecutionLogs.endedAt), + timeCursorStabilityBound(jobExecutionLogs.endedAt), cursorClause ) ) diff --git a/apps/sim/lib/data-drains/sources/workflow-logs.ts b/apps/sim/lib/data-drains/sources/workflow-logs.ts index b64ce154791..4466c5a4c60 100644 --- a/apps/sim/lib/data-drains/sources/workflow-logs.ts +++ b/apps/sim/lib/data-drains/sources/workflow-logs.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { workflowExecutionLogs } from '@sim/db/schema' import { and, inArray, isNotNull } from 'drizzle-orm' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' @@ -7,6 +7,7 @@ import { encodeTimeCursor, timeCursorOrderBy, timeCursorPredicate, + timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' @@ -33,13 +34,14 @@ async function* pages(input: SourcePageInput): AsyncIterable { cursor ) - const rows = await db + const rows = await dbReplica .select() .from(workflowExecutionLogs) .where( and( inArray(workflowExecutionLogs.workspaceId, workspaceIds), isNotNull(workflowExecutionLogs.endedAt), + timeCursorStabilityBound(workflowExecutionLogs.endedAt), cursorClause ) ) diff --git a/apps/sim/lib/logs/list-logs.test.ts b/apps/sim/lib/logs/list-logs.test.ts index 95fdda6bea0..b80bde60a9b 100644 --- a/apps/sim/lib/logs/list-logs.test.ts +++ b/apps/sim/lib/logs/list-logs.test.ts @@ -6,11 +6,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { selectMock } = vi.hoisted(() => ({ selectMock: vi.fn() })) -vi.mock('@sim/db', () => ({ - db: { - select: selectMock, - }, -})) +vi.mock('@sim/db', () => { + const instance = { select: selectMock } + return { db: instance, dbReplica: instance } +}) // Local drizzle-orm mock: the global mock's `sql` lacks `.as()` and the chain // mock doesn't support `.orderBy().limit()`. We only need condition/sql builders diff --git a/apps/sim/lib/logs/list-logs.ts b/apps/sim/lib/logs/list-logs.ts index 131092fcd6f..0165d60ad73 100644 --- a/apps/sim/lib/logs/list-logs.ts +++ b/apps/sim/lib/logs/list-logs.ts @@ -1,4 +1,4 @@ -import { db } from '@sim/db' +import { dbReplica } from '@sim/db' import { jobExecutionLogs, pausedExecutions, @@ -186,7 +186,7 @@ export async function listLogs(params: ListLogsParams, userId: string): Promise< levelList.length > 0 && !levelList.some((l) => l === 'error' || l === 'info') const includeJobLogs = !hasWorkflowSpecificFilters && !triggersExcludeJobs && !levelExcludesJobs - const workflowQuery = db + const workflowQuery = dbReplica .select({ id: workflowExecutionLogs.id, workflowId: workflowExecutionLogs.workflowId, @@ -313,7 +313,7 @@ export async function listLogs(params: ListLogsParams, userId: string): Promise< } const jobQuery = includeJobLogs - ? db + ? dbReplica .select({ id: jobExecutionLogs.id, executionId: jobExecutionLogs.executionId, diff --git a/packages/db/.env.example b/packages/db/.env.example index 14459e61f51..2f758f81173 100644 --- a/packages/db/.env.example +++ b/packages/db/.env.example @@ -1,5 +1,11 @@ # Database connection used by @sim/db scripts (drizzle-kit generate, # db:migrate, register-sso-provider, etc.). Must match DATABASE_URL in -# apps/sim/.env and apps/realtime/.env. +# apps/sim/.env and apps/realtime/.env. Migrations always run against the +# primary — never set a replica URL here. DATABASE_URL="postgresql://postgres:postgres@localhost:5432/simstudio" + +# Direct (non-pooled) DSN for db:migrate. Required when DATABASE_URL points at +# a transaction-pooling PgBouncer: session advisory locks and session SETs are +# unsupported through transaction pooling. Falls back to DATABASE_URL. +# MIGRATION_DATABASE_URL="" diff --git a/packages/db/db.ts b/packages/db/db.ts index 9e5597fb57b..397e11a894c 100644 --- a/packages/db/db.ts +++ b/packages/db/db.ts @@ -7,12 +7,30 @@ if (!connectionString) { throw new Error('Missing DATABASE_URL environment variable') } -const postgresClient = postgres(connectionString, { +const poolOptions = { prepare: false, idle_timeout: 20, connect_timeout: 30, - max: 15, onnotice: () => {}, -}) +} + +const postgresClient = postgres(connectionString, { ...poolOptions, max: 15 }) export const db = drizzle(postgresClient, { schema }) + +/** + * Opt-in read-replica client for reads that tolerate bounded staleness and have + * no read-your-writes dependency (logs, exports, dashboard aggregations). Never + * for auth, workflow state, or billing enforcement. Falls back to the primary + * when `DATABASE_REPLICA_URL` is unset, so call sites never branch. + */ +const replicaUrl = process.env.DATABASE_REPLICA_URL +if (replicaUrl && !/^postgres(ql)?:\/\//.test(replicaUrl)) { + throw new Error( + 'DATABASE_REPLICA_URL is set but is not a postgres:// DSN — fix or unset it (reads fall back to the primary when unset)' + ) +} + +export const dbReplica: typeof db = replicaUrl + ? drizzle(postgres(replicaUrl, { ...poolOptions, max: 10 }), { schema }) + : db diff --git a/packages/db/scripts/migrate.ts b/packages/db/scripts/migrate.ts index ed0af3b1c4d..4211001e670 100644 --- a/packages/db/scripts/migrate.ts +++ b/packages/db/scripts/migrate.ts @@ -1,68 +1,80 @@ +import { getPostgresErrorCode } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' import { drizzle } from 'drizzle-orm/postgres-js' import { migrate } from 'drizzle-orm/postgres-js/migrator' import postgres from 'postgres' /** - * Concurrent-index convention (avoid write-blocking index builds on large tables) - * -------------------------------------------------------------------------------- - * drizzle-kit emits plain `CREATE INDEX`, which takes a SHARE lock and blocks all - * writes for the build duration — on a big, write-hot table (e.g. - * workflow_execution_logs, usage_log) that stalls every in-flight workflow - * completion for minutes. drizzle wraps each migration in a transaction, and - * `CREATE INDEX CONCURRENTLY` cannot run inside a transaction block. - * - * So, after generating a migration that adds an index on a large/hot table, edit - * the generated SQL to end drizzle's transaction first, then build concurrently - * and idempotently: + * Concurrent-index convention: plain `CREATE INDEX` write-blocks large/hot + * tables, and CONCURRENTLY cannot run inside drizzle's migration transaction. + * For indexes on big tables, edit the generated SQL to: * * COMMIT;--> statement-breakpoint - * CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_name" ON "table" (...); + * SET lock_timeout = 0;--> statement-breakpoint + * CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_name" ON "table" (...);--> statement-breakpoint + * SET lock_timeout = '5s'; * - * Notes: - * - Put the `COMMIT` breakpoint AFTER all transactional DDL (ALTER TABLE/TYPE) - * in the file and only the concurrent CREATE INDEX statements below it. - * - Use `IF NOT EXISTS` (and make sibling DDL idempotent, e.g. - * `ADD COLUMN IF NOT EXISTS`, `ADD VALUE IF NOT EXISTS`) so a re-run after a - * failed CONCURRENTLY build is safe — fresh DBs and re-applies both work. - * - CONCURRENTLY only takes a SHARE UPDATE EXCLUSIVE lock (allows reads/writes). - * - Always validate on staging before prod; a failed CONCURRENTLY build can - * leave an INVALID index that must be dropped and rebuilt. + * The embedded COMMIT ends the batch transaction, so everything after it (in + * this and later pending files) runs in autocommit and must be idempotent + * (`IF NOT EXISTS` etc.) — a failed run replays unjournaled files from the top. + * A failed CONCURRENTLY build leaves an INVALID index that `IF NOT EXISTS` + * skips; `warnOnInvalidIndexes` below surfaces those. */ -const url = process.env.DATABASE_URL +/** + * Prefer a direct (non-pooled) DSN: session advisory locks and session `SET`s + * are unsupported through PgBouncer transaction pooling. Falls back to + * DATABASE_URL for setups that connect directly anyway. + */ +const url = process.env.MIGRATION_DATABASE_URL || process.env.DATABASE_URL if (!url) { console.error('ERROR: Missing DATABASE_URL environment variable.') console.error('Ensure packages/db/.env is configured.') process.exit(1) } -const client = postgres(url, { max: 1, connect_timeout: 10 }) +/** + * The pid guard is only sound on a direct connection — through transaction + * pooling, consecutive statements legitimately land on different backends. + */ +const hasDirectMigrationUrl = Boolean(process.env.MIGRATION_DATABASE_URL) /** - * Cross-process migration lock key (a stable, app-wide 64-bit constant). - * - * drizzle's `migrate()` has no built-in lock, so when a deployment starts N app - * replicas at once — each with a migration sidecar — all N read - * `__drizzle_migrations`, all see the same migration pending, and all try to apply - * it concurrently. One wins; the losers run the same DDL against already-mutated - * state and die (e.g. `DROP TABLE "form"` → `table "form" does not exist`, - * exit 1 / TaskFailedToStart). - * - * A session-level `pg_advisory_lock` serializes runners: the first to acquire it - * migrates while the rest block, then each loser acquires the lock, re-reads - * `__drizzle_migrations`, finds nothing pending, and exits cleanly. Session locks - * auto-release if the connection drops, so a crashed runner never wedges the lock. + * `max_lifetime: null` pins the session for the whole run: the postgres-js + * default recycles the connection after 30–60 min, silently dropping the + * session advisory lock and `SET`s. + */ +const client = postgres(url, { max: 1, connect_timeout: 10, max_lifetime: null }) + +/** + * Cross-process migration lock. drizzle's `migrate()` has no built-in lock, so + * concurrent runners (one per app replica at deploy time) must be serialized. + * Acquisition is a bounded try-lock loop: a plain `pg_advisory_lock` wait let + * one wedged runner silently hang every other runner and the whole deploy. */ const MIGRATION_LOCK_KEY = 4_961_002_270n +const LOCK_ACQUIRE_DEADLINE_MS = 30 * 60_000 +const LOCK_RETRY_INTERVAL_MS = 5_000 + +/** + * Max time a migration statement may queue for a table lock (SQLSTATE 55P03 on + * expiry). Without it, DDL waiting on an AccessExclusiveLock queues every other + * query on the table behind it — a table-wide stall for the whole wait. + */ +const DDL_LOCK_TIMEOUT = '5s' +const MAX_MIGRATE_ATTEMPTS = 8 +const MIGRATE_RETRY_BACKOFF = { baseMs: 2_000, maxMs: 30_000 } as const + +/** Backend pid of the lock-holding session; a change means the lock was lost. */ +let lockSessionPid = 0 try { - // statement_timeout=0: index builds (esp. CONCURRENTLY on large tables) can run - // far longer than the app default; a migration must never be killed mid-build. - await client`SET statement_timeout = 0` - await client`SELECT pg_advisory_lock(${MIGRATION_LOCK_KEY})` + await acquireMigrationLock() try { - await migrate(drizzle(client), { migrationsFolder: './migrations' }) + await runMigrationsWithRetry() console.log('Migrations applied successfully.') + await warnOnInvalidIndexes() } finally { await releaseMigrationLock() } @@ -75,11 +87,92 @@ try { } /** - * Release the advisory lock without ever failing the process. The session-level - * lock auto-releases when the connection closes, so a thrown unlock — e.g. the - * connection dropped right after `migrate()` committed — must be swallowed. - * Letting it reach the outer `catch` would exit 1 and falsely report a - * successful migration as failed to the deploy orchestrator. + * Acquire the cross-process migration lock, failing loudly after the deadline + * instead of blocking forever behind a wedged runner. + */ +async function acquireMigrationLock(): Promise { + const deadline = Date.now() + LOCK_ACQUIRE_DEADLINE_MS + for (;;) { + const [{ locked, pid }] = + await client`SELECT pg_try_advisory_lock(${MIGRATION_LOCK_KEY}) AS locked, pg_backend_pid() AS pid` + if (locked) { + lockSessionPid = pid + return + } + if (Date.now() >= deadline) { + throw new Error( + `Timed out after ${LOCK_ACQUIRE_DEADLINE_MS}ms waiting for the migration advisory lock; ` + + 'another runner is likely stuck mid-migration. Investigate before retrying.' + ) + } + await sleep(LOCK_RETRY_INTERVAL_MS) + } +} + +/** + * Run pending migrations, retrying on lock timeout (55P03, found anywhere in + * the wrapped `cause` chain). Each attempt re-verifies the lock session (pid) + * and re-asserts the session timeouts — a migration file may have changed them, + * and `SET` cannot be parameterized, hence `client.unsafe` with constants. + * Replays are safe: drizzle rolls the batch back on failure, and post-COMMIT + * CONCURRENTLY statements are idempotent by convention. + */ +async function runMigrationsWithRetry(): Promise { + for (let attempt = 1; ; attempt++) { + if (hasDirectMigrationUrl) { + const [{ pid }] = await client`SELECT pg_backend_pid() AS pid` + if (pid !== lockSessionPid) { + throw new Error( + `Database session changed mid-run (backend pid ${lockSessionPid} -> ${pid}); ` + + 'the migration advisory lock was lost. Aborting so a fresh runner can retry safely.' + ) + } + } + await client.unsafe('SET statement_timeout = 0') + await client.unsafe(`SET lock_timeout = '${DDL_LOCK_TIMEOUT}'`) + try { + await migrate(drizzle(client), { migrationsFolder: './migrations' }) + return + } catch (error) { + const isLockTimeout = getPostgresErrorCode(error) === '55P03' + if (!isLockTimeout || attempt >= MAX_MIGRATE_ATTEMPTS) throw error + const delayMs = backoffWithJitter(attempt, null, MIGRATE_RETRY_BACKOFF) + console.warn( + `WARN: migration DDL hit lock_timeout (attempt ${attempt}/${MAX_MIGRATE_ATTEMPTS}); ` + + `retrying in ${Math.round(delayMs)}ms.` + ) + await sleep(delayMs) + } + } +} + +/** + * A failed CONCURRENTLY build leaves an INVALID index that `IF NOT EXISTS` + * silently skips forever — surface it (warn only; the migration committed). + */ +async function warnOnInvalidIndexes(): Promise { + try { + const rows = await client` + SELECT n.nspname AS schema, c.relname AS index + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE NOT i.indisvalid + ` + for (const row of rows) { + console.warn( + `WARN: invalid index ${row.schema}.${row.index} — a CONCURRENTLY build failed partway. ` + + 'Drop and rebuild it; IF NOT EXISTS will keep skipping it.' + ) + } + } catch (checkError) { + console.warn('WARN: could not check for invalid indexes.', checkError) + } +} + +/** + * Unlock errors are swallowed: the session lock auto-releases on disconnect, + * and a thrown unlock would falsely report a committed migration as failed. */ async function releaseMigrationLock(): Promise { try { diff --git a/packages/testing/src/mocks/database.mock.ts b/packages/testing/src/mocks/database.mock.ts index f055746753e..7c7b9d6b489 100644 --- a/packages/testing/src/mocks/database.mock.ts +++ b/packages/testing/src/mocks/database.mock.ts @@ -235,17 +235,21 @@ export function resetDbChainMock(): void { * vi.mock('@sim/db', () => dbChainMock) * ``` */ +const dbChainInstance = { + select, + selectDistinct, + selectDistinctOn, + insert, + update, + delete: del, + execute, + transaction, +} + export const dbChainMock = { - db: { - select, - selectDistinct, - selectDistinctOn, - insert, - update, - delete: del, - execute, - transaction, - }, + db: dbChainInstance, + /** Same instance as `db` so per-test chain overrides cover both clients. */ + dbReplica: dbChainInstance, } /** @@ -309,8 +313,12 @@ export function createMockDb() { * vi.mock('@sim/db', () => databaseMock) * ``` */ +const mockDbInstance = createMockDb() + export const databaseMock = { - db: createMockDb(), + db: mockDbInstance, + /** Same instance as `db` so per-test overrides cover both clients. */ + dbReplica: mockDbInstance, sql: createMockSql(), ...createMockSqlOperators(), } From 167ec5ef1ead836c5b5dbe6b41f269d7a7e28f2c Mon Sep 17 00:00:00 2001 From: Waleed Date: Wed, 10 Jun 2026 18:35:39 -0700 Subject: [PATCH 4/7] fix(oauth): drop ungrantable JSM Forms scopes from Jira scope list (#4960) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(oauth): drop ungrantable JSM Forms scopes from Jira scope list Atlassian never published read/write/delete:form:jira-service-management to the OAuth 2.0 (3LO) or Forge scope catalogs, so no OAuth app can be configured with them and the authorize flow silently omits them from every grant. Because the credential check hard-requires the full canonical list, every Jira credential showed a permanent 'Additional permissions required' banner that 'Update access' could never clear. No granted credential has ever held these scopes, and no saved workflow uses the JSM forms operations, so removal changes no working behavior. * fix(secrets): keep a fixed-length value mask for read-only viewers The viewer mask was derived from the value's length, but the server now withholds workspace secret values from non-admins (empty string), so the bullets disappeared entirely for read-only users. Always render a fixed-length mask for viewers — matching the component's documented behavior — which also stops leaking the secret's length. --- .../components/secret-value-field/secret-value-field.tsx | 9 ++++++++- apps/sim/lib/oauth/oauth.ts | 3 --- apps/sim/lib/oauth/utils.ts | 3 --- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secret-value-field/secret-value-field.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secret-value-field/secret-value-field.tsx index fd3d8056c93..be6b14d6f9e 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secret-value-field/secret-value-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/secrets/components/secret-value-field/secret-value-field.tsx @@ -6,6 +6,13 @@ import { ChipInput } from '@/components/emcn' const BULLET = '\u2022' +/** + * Viewers always see this many bullets regardless of the real value, which the + * server withholds (empty string) for non-admins. A fixed length also avoids + * leaking the secret's length. + */ +const VIEWER_MASK_LENGTH = 10 + type SecretValueFieldProps = Omit< ComponentProps<'input'>, 'type' | 'value' | 'onChange' | 'readOnly' @@ -50,7 +57,7 @@ export function SecretValueField({ const [focused, setFocused] = useState(false) const editable = canEdit && !readOnly const maskActive = canEdit && !unmasked && !focused - const displayValue = canEdit ? value : value ? BULLET.repeat(value.length) : '' + const displayValue = canEdit ? value : BULLET.repeat(VIEWER_MASK_LENGTH) const mergedStyle: CSSProperties | undefined = maskActive ? ({ ...style, WebkitTextSecurity: 'disc' } as CSSProperties) diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index f7af93832e1..9cca712769f 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -541,9 +541,6 @@ export const OAUTH_PROVIDERS: Record = { 'write:request.participant:jira-service-management', 'read:request.approval:jira-service-management', 'write:request.approval:jira-service-management', - 'read:form:jira-service-management', - 'write:form:jira-service-management', - 'delete:form:jira-service-management', ], }, }, diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index 1f7eae2e774..f95626781a2 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -201,9 +201,6 @@ export const SCOPE_DESCRIPTIONS: Record = { 'Add and remove participants from customer requests', 'read:request.approval:jira-service-management': 'View approvals on customer requests', 'write:request.approval:jira-service-management': 'Approve or decline customer requests', - 'read:form:jira-service-management': 'View JSM forms and templates', - 'write:form:jira-service-management': 'Attach, save, and submit JSM forms', - 'delete:form:jira-service-management': 'Delete JSM forms', // Microsoft scopes 'User.Read': 'Read Microsoft user', From bc55fc3b50933cbdc46844b0c8207a99ec4e6b31 Mon Sep 17 00:00:00 2001 From: Will Chen Date: Wed, 10 Jun 2026 18:39:57 -0700 Subject: [PATCH 5/7] improvement(docs): builder-first IA reorganization of the English docs (#4896) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: reorganize into topic/ontology IA with a builder-first rewrite Restructure the English docs from internal product categories into a topic-based information architecture, and rewrite the conceptual pages to install a mental model first rather than enumerate features. Structure & navigation - Reorder the sidebar to follow how someone builds: Get Started -> Workflows -> Tables -> Files -> Knowledge Bases -> Logs -> Building agents -> Mothership -> Workspaces -> Platform -> Reference. - Demote the generated blocks/tools/triggers catalogs to a Reference section at the bottom. - Break up the monolithic execution/ folder into deployment/ and logs-debugging/; collapse connections/* and variables/* into single pages under workflows/. - Rename capabilities/ to building-agents/; relabel the integration catalog as "Integrations". Remove deprecated copilot and form deployment. Redirects added in next.config.ts for every moved URL. Conceptual rewrites - Workflows core (index, how-it-runs, data-flow, connections, variables): one mental model, one running example, terser prose. - New building-agents overview distinguishes an agent (a workflow you build) from an Agent block (one reasoning step), plus a "choosing what to use" guide. - Concept-trim passes on Knowledge Base, Tables, Blocks, Triggers overviews; new task pages for KB, Tables, and Files. - New code-verified Alerts page. Infrastructure - pageType frontmatter (concept/guide/reference) + badge render. - WorkflowPreview / OutputBundle components to embed real, app-styled workflow diagrams (adds framer-motion + reactflow to apps/docs). Co-Authored-By: Claude Opus 4.8 (1M context) * feat(docs): spec-driven BlockPreview for block reference heroes Replace the static screenshot hero on each block reference page with a that renders the block exactly as the builder canvas shows it — header icon, sub-block rows, and branch/error handles — from a hand-authored display spec. Static and non-interactive (no ReactFlow), so it can't be panned or dragged, and self-updating to edit. - block-display-specs.ts: one editable spec per block (rows, branches, handles) - block-preview.tsx: static scaled card renderer with decorative handles - block-icons.tsx: brand glyphs for the core block types; icons.tsx adds WaitIcon - 14 block + 3 trigger pages swapped from to Co-Authored-By: Claude Opus 4.8 (1M context) * fix(docs): correct stale navigation and removed-feature references Audited the docs against the product changelog (GitHub releases / staging git history) for content that misleads readers — features that moved, were renamed, or removed — rather than cosmetic drift. Fixes: - Skills: no longer a Settings tab. It was promoted to its own workspace page (#4354), so "Settings → Skills under the Tools section" sent readers to a tab that no longer exists. (skills/index.mdx) - Env vars: the workspace tab is "Secrets", not "Environment Variables" (credentials→secrets rename, #4364). (quick-reference/index.mdx) - Mothership FAQ pointed to "Settings → Credentials" for integration connections; integrations moved to their own page and there is no Credentials tab. (mothership/tasks.mdx) - Vision block was retired (#4684); a tip still named it. Reworded to "an Agent using a vision-capable model". (files/passing-files.mdx) - Getting-started FAQ told new users to "use the Copilot feature" to build in natural language — that surface is Mothership. (getting-started) - Removed the dead "Mod+Y → Go to templates" shortcut; the templates gallery was removed (#4354). (keyboard-shortcuts) Note: MCP "tools" (Settings → Tools, for consuming) and MCP "servers" (Settings → System, for exposing) are distinct surfaces — both doc references are correct and were intentionally left as-is. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(docs): repair broken /docs-prefixed enterprise links The enterprise overview linked to /docs/enterprise/* (access-control, sso, whitelabeling, audit-logs, data-retention, data-drains), but the docs site is served at root — those 6 links 404'd. Now root-relative /enterprise/*. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(docs): refresh stale workflow-preview example blocks The /workflows diagram blocks are hand-authored (separate from the spec-driven BlockPreview heroes) and had drifted from the real UI: - Agent color purple #6f3dfa -> green #33C482 (the var(--brand) rebrand) - Model gpt-4o -> claude-sonnet-4-6 (current default) - "Prompt" row -> "Messages" (the actual agent sub-block) - Start color #34B5FF -> #2FB3FF (real starter bgColor) Co-Authored-By: Claude Opus 4.8 (1M context) * fix(docs): align BlockPreview input/output handles to the card edge The header (input/output) handles are positioned relative to the card and used a -16px offset, so they floated 8px past the edge. Row/error handles are -16px relative to a row that's already inset 8px by content padding, so they sit correctly. Header handles are now -8px, so every handle sticks out the same 8px and hugs the block edge. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Agent reference to match the current block The page documented the old UI (System/User Prompt, no Files or Skills, Memory taught as a separate block — contradicting its own FAQ). Rewritten to the real sub-blocks (Messages, Model, Files, Tools, Skills, Memory, Response Format) in the builder voice of the workflows exemplars: oriented opening, agent vs Agent-block callout, outputs table, a live WorkflowPreview example, FAQ kept and corrected (tool control "Force", not "Required"). pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite API reference to match the current block Tightened to the builder voice and the real config (URL, Method, Query Params, Headers, Body + Advanced timeout/retries/backoff). Dropped the off-topic "Dynamic URL Construction" / "Response Validation" sections (those are Function-block techniques, not API config). Outputs table, FAQ kept. The example is now a live WorkflowPreview (new API_FETCH_WORKFLOW in examples.ts, exported via the barrel). pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Condition reference to match the current block Tightened to the builder voice: oriented opening (branches on boolean expressions, no model call, vs Router), the real branch model (if / else if / else, checked top to bottom), connection-tag expression examples, an error-path callout, outputs table, and a live branching WorkflowPreview example (CONDITION_ROUTE_WORKFLOW). FAQ kept. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): restore Best Practices + multi-example workflows on Condition Recalibration: reference pages keep genuine substance (Best Practices, every distinct example), cutting only redundancy and verbose register. Restores the Best Practices section and turns the three use cases into three rendered WorkflowPreview examples (route by priority, moderate content, branch onboarding). Adds CONDITION_MODERATE_WORKFLOW and CONDITION_ONBOARD_WORKFLOW. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): restore Best Practices on Agent reference Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): restore Best Practices on API reference Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Function reference to match the current block Fixed the verbose register and dropped the duplicated outputs section + the stale Python screenshot/TODO, while keeping the real substance: JS vs Python (local vs E2B sandbox), the large-inputs sim.files/sim.values helpers, the worked loyalty-score example, and Best Practices. The use cases are now two rendered WorkflowPreview examples (reshape an API response, validate input). Adds FUNCTION_RESHAPE_WORKFLOW and FUNCTION_VALIDATE_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Router reference to match the current block Cleaned the register, generalized the drifting model list, and folded the Router-vs-Condition guidance into a callout. Kept the substance (routes as output ports, NO_MATCH error path, all seven outputs, Best Practices, FAQ). The three same-shape use cases collapse to one rendered triage WorkflowPreview (ROUTER_TRIAGE_WORKFLOW), which the prose notes stands for the pattern. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): restore the classify and lead-qual examples on Router I wrongly folded two distinct Router scenarios into a note. Restored all three as their own rendered WorkflowPreview examples: triage a support ticket, classify feedback (to child workflows), qualify a lead (sales vs self-serve). Adds ROUTER_CLASSIFY_WORKFLOW and ROUTER_LEAD_WORKFLOW. (Also exports RESPONSE_API_WORKFLOW for the next page.) Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Response reference to match the current block Cleaned the register and broadened "Variable References" to connection tags (any output, not just workflow variables). Kept the substance: exit-point semantics, Builder/Editor mode, status codes, headers, the parallel-branch warning, Best Practices, FAQ. All three use cases are now rendered WorkflowPreview examples (API endpoint, webhook ack, status-per-branch). Adds RESPONSE_API/WEBHOOK/ERROR_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Variables reference to match the current block Cleaned the register, corrected the outputs (each assignment is also exposed as , not "no outputs"), and kept the substance: assignments reference earlier outputs and current values, global access, Best Practices, FAQ. Two use cases now render as WorkflowPreview examples (count retries, hold config). Adds VARIABLES_RETRY_WORKFLOW and VARIABLES_CONFIG_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Wait reference to match the current block Corrected a real staleness: the block now has an Async mode that suspends the run for minutes/hours/days (not a hard 10-minute cap), plus a resumeAt output. Documents Wait Amount / Unit / Async, the sync-vs-async distinction, all three outputs, Best Practices, and updated FAQ. Two rendered WorkflowPreview examples (space out API calls, delayed follow-up). Adds WAIT_RATELIMIT_WORKFLOW and WAIT_FOLLOWUP_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): polish Credential reference (frontmatter, fold redundant tabs) The page was already accurate to the block (Select/List operations, the outputs tabs, the wiring steps). Light touch only: added description + pageType, made the header consistent, and folded the two identical Gmail/Slack "how to wire" tabs into one line. Examples stay as labeled flows + the List/ForEach screenshot, since they use integration blocks and a Loop the WorkflowPreview can't render. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): render the shared-credential example + icon fallback for integrations Addressing the gap: WorkflowPreview block nodes now fall back to the integration icon map, so diagrams can show Gmail/Drive/Slack/etc. with their real glyphs, not just core blocks. Renders the Credential "share one account across blocks" example as a WorkflowPreview (CREDENTIAL_SHARE_WORKFLOW). The multi-account and List+ForEach examples stay as labeled flows + screenshot (the latter uses a Loop container the preview can't render). Also exports EVALUATOR_GATE_WORKFLOW for the next page. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): rewrite Evaluator reference to match the current block Cleaned the register, generalized the drifting model list, and documented the per-metric outputs (), which the page omitted. Kept the substance (metrics with name/description/range, structured-output guarantee, Best Practices, FAQ). The quality-gate example renders as a WorkflowPreview; the same shape covers the parallel-variations and support-QC patterns, noted in prose. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): render the Credential route-by-logic example too The icon fallback unblocked it: the "route to a different account by logic" example now renders as a WorkflowPreview (CREDENTIAL_ROUTE_WORKFLOW), a Condition selecting a production vs staging credential. The List + ForEach example stays a screenshot because it nests blocks in a Loop container the flat WorkflowPreview can't represent. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): render Guardrails examples + light accuracy pass Kept the full substance (four validation types, PII entity/language detail, the PII screenshot and video, outputs, Best Practices, FAQ). Light fixes: frontmatter, and generalized the drifting model names (GPT-4o / Claude 3.7) to "a strong reasoning model" with the current default. The three use cases now render as WorkflowPreview examples (validate JSON, check grounding, block PII). Adds GUARDRAILS_JSON/HALLUCINATION/PII_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): render Human-in-the-Loop examples + frontmatter Kept all the substance (Display Data, Notification, Resume Form, the Approval Methods and API Execute Behavior tabs, outputs, the paused/resume example). Added frontmatter and rendered the use cases as WorkflowPreview examples (approve before publish, two-stage approval, verify extracted data); Quality Control folds into the approval note as the same approve-then-act shape. Adds HITL_APPROVAL/MULTISTAGE/VALIDATE_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): render Webhook examples + frontmatter The page was already accurate (Webhook URL/Payload/Signing Secret/Headers, the automatic-headers table, HMAC details, outputs, POST-only callout, FAQ). Added frontmatter and rendered the two use cases as WorkflowPreview examples (notify a service, fire on a check). Adds WEBHOOK_NOTIFY_WORKFLOW and WEBHOOK_TRIGGER_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): add example + pageType to Workflow block reference The page was already accurate and well-structured (Configure It, outputs, deployment-status badge, execution notes, FAQ). Added pageType: reference and a rendered WorkflowPreview example showing a parent calling the child workflow enrich-lead and reading its result. Adds WORKFLOW_CALL_WORKFLOW. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): container rendering for Loop/Parallel + render the Loop example Adds subflow/container support to WorkflowPreview, modeled on the app's subflow-node.tsx: a solid-bordered box with a header (icon + name), an internal "Start" pill whose handle feeds the first nested block, and target/source handles at the vertical center. PreviewBlock gains size/parentId; edges gain an optional sourceHandle; nodes render nested children via React Flow parentNode. Renders the Loop reference's ForEach example (LOOP_WORKFLOW) and keeps the four loop-type sections + inside/outside referencing + caps. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): fix the Loop container's Start-pill connector The Start pill -> first-block edge wasn't rendering: it was a React Flow parent->child edge (unreliable), and the opaque container body hid it. Nested blocks now render as absolute-positioned top-level nodes (container below at zIndex 0, blocks above at zIndex 1), so the connector is an ordinary edge, and the container body is see-through so it's visible. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): render the Parallel example + frontmatter (last core block) Reuses the container rendering for the Parallel reference. Kept all substance (count/collection types, inside/outside referencing, batch size of 20, instance isolation, the Parallel-vs-Loop table, Best Practices, FAQ). Added frontmatter and a rendered container WorkflowPreview (PARALLEL_WORKFLOW: distribute tasks, call concurrently, aggregate ); the two use cases stay as labeled flows. Adds PARALLEL_WORKFLOW. pageType: reference. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): prose glow-up for Guardrails to match the agent/condition voice Rewrote the listy register (**Use Cases:** / **How It Works:** / **Configuration:** scaffolding, "Use this when you need to..." filler) into the plain builder voice, matching the depth of the Agent/Condition/Function rewrites. Kept every validation type, option, range, the full PII entity/region list, the screenshot and video, the outputs table, the rendered examples, Best Practices, and FAQ. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): prose glow-up for Loop to match the agent/condition voice Rewrote into the plain builder voice and cut the filler: dropped the "Use this when you need to..." lines and the ASCII "Example: Iteration 1, 2, 3" pseudo-code, and folded the duplicated Inputs/Outputs tabs into Configuration + Referencing sections. Kept all four loop types with their screenshots, the inside/outside reference rules, the 1,000-iteration cap, sequential-vs-parallel guidance, the rendered example, and FAQ. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): prose glow-up for Parallel to match the agent/condition voice Same treatment as Loop: plain builder voice, dropped the ASCII pseudo-code and the duplicated Inputs/Outputs tabs, folded the verbose Advanced Features into tight Configuration + Referencing sections. Kept both types with screenshots, the batch-size-of-20 cap, instance isolation, large-result indexing, the Parallel-vs-Loop table, the rendered example, and FAQ. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): prose glow-up for Human-in-the-Loop Tightened the register: folded the pause sentence into the intro, made the section headers consistent (Configuration, Outputs), converted the bold-list Block Outputs into a table, condensed the Notification channel bullets to a line, and renamed the second "Example" so it no longer collides with the rendered Examples. Kept all the substance — Display Data / Notification / Resume Form, the Approval Methods and API Execute Behavior tabs, the portal video, and FAQ. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): re-enrich Loop prose (fuller, explanatory — not terse) The first glow-up overcorrected into terse fragments. Restored proper docs-quality prose at the Agent/Condition level: each loop type now explains what it does, when to use it, and the relevant reference; Configuration, Referencing, nesting, and Best Practices give context and the "why," not just bullets. Same substance, readable depth. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): re-balance Parallel prose to the Agent/Condition register Calibrated to the level signed off on elsewhere: each concept explained in a couple of clear sentences with a concrete detail — informative, not terse, not padded. Kept both types with screenshots, batch-size cap, isolation, large-result indexing, the Parallel-vs-Loop table, the rendered example, and FAQ. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): restore the Notification channel detail on HITL The glow-up over-compressed: it flattened the five notification channels (each with what they do) into one sentence. Restored them as a list in plain voice — tightening register shouldn't drop genuinely useful reference detail. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(blocks): builder-voice polish on the Credential intro Light touch only — the page was already well-structured and explanatory, so just led the intro with what the block does (and bolded the name) to match the other references. No content changed elsewhere. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(triggers): rewrite Start trigger in the builder voice Tightened the register, swapped the <> noise for backticks, added pageType + an outputs table, and kept all substance: Input Format types, chat-only outputs (input/conversationId/files), the editor/API/chat tabs, and best practices. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(triggers): rewrite Schedule trigger in the builder voice Plain voice and clean markdown (dropped the raw