diff --git a/.oxlintrc.effect.json b/.oxlintrc.effect.json index 9af397b453..b987aae199 100644 --- a/.oxlintrc.effect.json +++ b/.oxlintrc.effect.json @@ -13,6 +13,13 @@ // Last match wins, so this stays after the entry above. "apps/cli/src/shared/compute/stacks/**", "!apps/cli/tests/helpers/compute.ts", - "!apps/cli/src/command-internal/experimental-feature.ts" + "!apps/cli/src/command-internal/experimental-feature.ts", + "!apps/cli/src/command-internal/postgres-client.run.ts", + "!apps/cli/src/command-internal/stack-api.ts", + "!apps/cli/src/command-internal/stack-backend.ts", + "!apps/cli/src/command-internal/stack-catalog-setup.ts", + "!apps/cli/src/command-internal/stack-config.ts", + "!apps/cli/src/command-internal/stack-local-database.ts", + "!apps/cli/src/command-internal/stack-shadow.ts" ] } diff --git a/AGENTS.md b/AGENTS.md index fd890afc1e..df21f86796 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,8 +18,10 @@ reference. Published `apps/cli` and `packages/config` are not private; `apps/doc root-owned. Effect lint covers `packages/stack`, all files under `apps/cli/src/commands/experimental/stack` and `apps/cli/src/commands/experimental/compute`, the shared `apps/cli/src/shared/compute` runtime helpers (excluding embedded starter templates), the -Compute test fixture helper, and `apps/cli/src/command-internal/experimental-feature.ts`; use the -root scripts for it. +Compute test fixture helper, and `apps/cli/src/command-internal/experimental-feature.ts`, +`postgres-client.run.ts`, `stack-api.ts`, `stack-backend.ts`, `stack-catalog-setup.ts`, +`stack-config.ts`, `stack-local-database.ts`, and `stack-shadow.ts`; use the root scripts for +it. ### Config Naming Vocabulary diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index c2382fd8fa..39b91ab458 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -39,8 +39,26 @@ command. For temporary selection, set `SUPABASE_EXPERIMENTAL_STACK=1` to select the new backend or `SUPABASE_EXPERIMENTAL_STACK=0` to select the legacy backend. This environment variable takes precedence over `experimental.stack`; an unset or empty value falls back to the file setting. -Other values are rejected. The override affects only the top-level lifecycle aliases and is -applied before reading the project configuration. +Other values are rejected. The override is applied before reading the project configuration. + +When the flag is on, the `db` and `migration` family uses the project stack for `--local` and +provisions throwaway shadow Postgres through `@supabase/stack` (`EphemeralPostgres`). Linked +and `--db-url` targets stay on the Management API. Compose names (`supabase_db_*`, +`supabase_network_*`, `db:5432`) are not used. The stack backend requires the in-process +pg-delta engine; `--use-migra`, `--use-pgadmin`, `--use-pg-schema`, and `--diff-engine migra` +are rejected. The flag does not switch functions or storage command families, and does not +change top-level `status`. + +`db start` brings up a postgres-only project stack. If a full stack already exists, it starts +the database without persisting `--exclude`. `--from-backup` is not supported on the stack +path. `db reset --local` and declarative `--apply` wipe Postgres through `resetDatabase` and +then migrate or seed on stack credentials. + +`db dump --local`, `db test` / `test db`, and `migration squash` use host `pg_dump` / `pg_prove` +only when the stack engine is native. Those PATH clients must match the stack Postgres major; +otherwise install matching client tools or start with `--runtime docker`. The Docker/Podman +engine keeps the one-shot tool container and targets published stack credentials, never +`PGHOST=db`. ## Data and configuration @@ -52,12 +70,16 @@ The flag is local CLI configuration in `supabase/config.toml` and is excluded fr configuration. Routing reads that exact file after applying the CLI's working-directory rules, including `--workdir` and `SUPABASE_WORKDIR`; a JSON-only project does not enable the flag. +## Port intents + +Host listener assignment for `supabase stack` is documented in [Port intents](./supabase-home.md#port-intents). + ## Service selection and shutdown `supabase stack start --exclude studio,analytics -x mail` disables those services in the effective start configuration without changing the project file. Valid names are `rest`, `auth`, `realtime`, `storage`, `functions`, `studio`, `mail`, `analytics`, and `pooler`; the database is required. -Excluding `rest` or `analytics` also disables Studio. The effective configuration is +Excluding `rest` also disables Studio; excluding `analytics` does not. The effective configuration is retained in stack state, so starting without `--exclude` restores the project's configured services. `supabase stack stop --all` stops every readable managed stack while preserving data. It continues diff --git a/apps/cli/src/cli/complete.unit.test.ts b/apps/cli/src/cli/complete.unit.test.ts index 92dd1bf716..9bee1c46ca 100644 --- a/apps/cli/src/cli/complete.unit.test.ts +++ b/apps/cli/src/cli/complete.unit.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { Cause } from "effect"; import { rootCommand } from "./root.ts"; -import { StackRoutingError } from "../commands/experimental/stack/stack-backend.ts"; +import { StackRoutingError } from "../command-internal/stack-backend.ts"; import { CompletionDirective, type ClassifyCompletionInput, diff --git a/apps/cli/src/cli/main.ts b/apps/cli/src/cli/main.ts index 01271f4bb2..972d154e36 100644 --- a/apps/cli/src/cli/main.ts +++ b/apps/cli/src/cli/main.ts @@ -5,7 +5,7 @@ import { runCli } from "../shared/cli/run.ts"; import { upgradeNoticeHook } from "../command-internal/upgrade-notice.ts"; import { analyticsLayer } from "../telemetry/analytics.layer.ts"; import { defaultCompleteDeps, tryComplete } from "./complete.ts"; -import { resolveStackBackend } from "../commands/experimental/stack/stack-backend.ts"; +import { resolveStackBackend } from "../command-internal/stack-backend.ts"; import { resolveComputeEnabled } from "../commands/experimental/compute/compute-backend.ts"; import { rootCommandForFeatures } from "./root.ts"; diff --git a/apps/cli/src/cli/root.ts b/apps/cli/src/cli/root.ts index 47e94b924c..1313deabf5 100644 --- a/apps/cli/src/cli/root.ts +++ b/apps/cli/src/cli/root.ts @@ -11,7 +11,8 @@ import { encryptionCommand } from "../commands/encryption/encryption.command.ts" import { stackRuntimeLayer, stackCommand } from "../commands/experimental/stack/stack.command.ts"; import { stackStartCommand } from "../commands/experimental/stack/start/start.command.ts"; import { stackStopCommand } from "../commands/experimental/stack/stop/stop.command.ts"; -import type { StackBackend } from "../commands/experimental/stack/stack-backend.ts"; +import type { StackBackend } from "../command-internal/stack-backend.ts"; +import { stackBackendLayer } from "../command-internal/stack-backend.ts"; import { computeCommand } from "../commands/experimental/compute/compute.command.ts"; import { feedbackCommand } from "../commands/feedback/feedback.command.ts"; import { functionsCommand } from "../commands/functions/functions.command.ts"; @@ -181,6 +182,7 @@ export const rootCommandForFeatures = ( : outputLayerFor(outputFormat); return Layer.mergeAll( + stackBackendLayer(options.stackBackend ?? "legacy"), outputLayer, makeGoProxyLayer({ globalArgs, parentOwnsCapturedSuccessTail: true }), ); diff --git a/apps/cli/src/command-internal/container-cli.ts b/apps/cli/src/command-internal/container-cli.ts index 9318b3dcd8..71d8cabe65 100644 --- a/apps/cli/src/command-internal/container-cli.ts +++ b/apps/cli/src/command-internal/container-cli.ts @@ -110,7 +110,7 @@ export const containerCliExitCode = ( ); /** Folds a byte stream into a decoded string. */ -export function collectText(stream: Stream.Stream) { +export function collectText(stream: Stream.Stream) { const decoder = new TextDecoder(); return Stream.runFold( stream, diff --git a/apps/cli/src/command-internal/db-bootstrap/db-setup.ts b/apps/cli/src/command-internal/db-bootstrap/db-setup.ts index 068f74d7cd..47f60afda4 100644 --- a/apps/cli/src/command-internal/db-bootstrap/db-setup.ts +++ b/apps/cli/src/command-internal/db-bootstrap/db-setup.ts @@ -705,6 +705,75 @@ export const startInitCurrentBranch = Effect.fnUntraced(function* ( ); }); +export interface ApplyDatabaseOverlayInput { + readonly webhooksEnabled: boolean; + readonly apiAutoExposeNewTables: Option.Option; + readonly vault: ReadonlyArray; + readonly webhooks?: SetupDatabaseOptions["webhooks"]; + /** When false, skip the roles.sql stderr banner used by live setup. */ + readonly announceRoles?: boolean; +} + +/** + * Session SQL after schema init: webhooks (`pg_net`), API default grants, vault upsert, `roles.sql`. + */ +export const applyDatabaseOverlay = ( + session: DbSession, + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, + overlay: ApplyDatabaseOverlayInput, +): Effect.Effect => + Effect.gen(function* () { + yield* Effect.scoped( + Effect.gen(function* () { + const tmpDir = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-db-overlay-" }).pipe( + Effect.mapError( + (error) => + new DbSetupError({ + message: `failed to create temp directory: ${errMessage(error)}`, + reason: "filesystem", + }), + ), + ); + yield* applyDatabaseWebhooks( + session, + fs, + path, + tmpDir, + resolveSetupWebhooksEnabled(overlay.webhooks, overlay.webhooksEnabled), + ); + yield* applyApiPrivileges(session, fs, path, tmpDir, overlay.apiAutoExposeNewTables); + }), + ); + + yield* upsertVaultSecrets(session, overlay.vault); + + const customRolesPath = path.join(workdir, "supabase", "roles.sql"); + if (overlay.announceRoles !== false) { + const output = yield* Output; + yield* output.raw(`Seeding globals from ${path.basename(customRolesPath)}...\n`, "stderr"); + } + const rolesExist = yield* fs.exists(customRolesPath).pipe( + Effect.mapError( + (error) => + new DbSetupError({ + message: `failed to check roles.sql: ${errMessage(error)}`, + reason: "filesystem", + }), + ), + ); + if (rolesExist) { + yield* execSqlFile( + session, + fs, + path, + customRolesPath, + (message) => new DbSetupError({ message, reason: "database" }), + ); + } + }); + /** * Runs schema init through the custom-roles seed; see {@link SetupDatabaseInput} for exactly * what's in and out of scope. Extracted from {@link startSetupLocalDatabase} so shadow-database @@ -745,44 +814,15 @@ export const setupDatabase = ( if (requiresPg14WebhooksCleanup) { yield* removeDatabaseWebhooks(session, fs, path, tmpDir); } - yield* applyDatabaseWebhooks( - session, - fs, - path, - tmpDir, - resolveSetupWebhooksEnabled(options.webhooks, input.webhooksEnabled), - ); - yield* applyApiPrivileges(session, fs, path, tmpDir, input.apiAutoExposeNewTables); }), ); - // Runs before the roles seed so `roles.sql` can reference these secrets. - yield* upsertVaultSecrets(session, input.vault); - - // Prints unconditionally, before checking whether the file exists. A missing file is - // tolerated; any other read/exec error propagates. Checked via an existence check ahead of - // the read rather than a caught not-found error — no meaningful TOCTOU concern here. - const customRolesPath = path.join(workdir, "supabase", "roles.sql"); - const output = yield* Output; - yield* output.raw(`Seeding globals from ${path.basename(customRolesPath)}...\n`, "stderr"); - const rolesExist = yield* fs.exists(customRolesPath).pipe( - Effect.mapError( - (error) => - new DbSetupError({ - message: `failed to check roles.sql: ${errMessage(error)}`, - reason: "filesystem", - }), - ), - ); - if (rolesExist) { - yield* execSqlFile( - session, - fs, - path, - customRolesPath, - (message) => new DbSetupError({ message, reason: "database" }), - ); - } + yield* applyDatabaseOverlay(session, fs, path, workdir, { + webhooksEnabled: input.webhooksEnabled, + apiAutoExposeNewTables: input.apiAutoExposeNewTables, + vault: input.vault, + webhooks: options.webhooks, + }); }); /** diff --git a/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts b/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts index 7196372509..02d1ef6e44 100644 --- a/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts +++ b/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts @@ -28,12 +28,20 @@ import { } from "../../shared/telemetry/error-actionability.ts"; import { aqua, yellow } from "../colors.ts"; import { CommandSettings } from "../../config/command-settings.service.ts"; -import { checkDbToml, loadProjectEnv } from "../db-config.toml-read.ts"; +import { checkDbToml, loadProjectEnv, readDbToml } from "../db-config.toml-read.ts"; +import { DbConnection } from "../db-connection.service.ts"; +import { loadLocalProjectContext } from "../local-project-context.ts"; +import { migrateAndSeed } from "../migrate-and-seed.ts"; import { seedBucketsRun } from "../seed-buckets.ts"; import { awaitStorageReady } from "./await-storage-ready.ts"; +import { resolveResetSeedConfig } from "./db-setup.ts"; import { buildLocalDbContainerInputs } from "./local-container-inputs.ts"; import { isLocalDbRunning } from "./local-db-running.ts"; import { recreateLocalDatabase } from "./recreate-local-database.ts"; +import { currentStackBackend } from "../stack-backend.ts"; +import { stackLocalDatabaseConn, stackOpenReadyProject } from "../stack-local-database.ts"; +import { loadStackConfig } from "../stack-config.ts"; +import { StackCatalogSetup } from "../stack-catalog-setup.ts"; /** The local database container is not running. */ class ResetLocalDbNotRunningError extends Data.TaggedError("ResetLocalDbNotRunningError")<{ @@ -44,6 +52,15 @@ class ResetLocalDbNotRunningError extends Data.TaggedError("ResetLocalDbNotRunni } } +class ResetLocalDbFailedError extends Data.TaggedError("ResetLocalDbFailedError")<{ + readonly message: string; + readonly suggestion?: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.dbConnection; + } +} + /** ` to version: X`, or `...` when resetting to the latest migration. */ const toLogMessage = (version: string): string => version.length > 0 ? ` to version: ${version}` : "..."; @@ -60,20 +77,22 @@ const PLAIN_FULL_RESET: ResetLocalDatabaseInput = { seedFlags: { noSeed: false, sqlPaths: [] }, }; +const notRunning = () => + new ResetLocalDbNotRunningError({ + message: `${aqua("supabase start")} is not running.`, + }); + +const resetFailed = (message: string) => new ResetLocalDbFailedError({ message }); + /** Resets the local database in-process. See this module's own header for the full design rationale. */ export const resetLocalDatabase = Effect.fnUntraced(function* ( input: ResetLocalDatabaseInput = PLAIN_FULL_RESET, ) { + const backend = yield* currentStackBackend; const output = yield* Output; const cliSettings = yield* CommandSettings; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const runtimeInfo = yield* RuntimeInfo; - const networkIdFlag = yield* NetworkIdFlag; - // Threaded into `buildLocalDbContainerInputs`'s `setup.debug`, so a failed fresh-volume - // Realtime/Storage/Auth migrate job on the PG15 recreate path tees its own stderr. - const debug = yield* DebugFlag; const workdir = cliSettings.workdir; // Load the project env first so a `SUPABASE_EXPERIMENTAL` set only in `supabase/.env` is @@ -82,10 +101,103 @@ export const resetLocalDatabase = Effect.fnUntraced(function* ( const yes = yield* resolveYesWithProjectEnv(projectEnv); const experimental = yield* resolveExperimentalWithProjectEnv(projectEnv); - // Validate config before checking whether the container is running, so a malformed config + // Validate config before checking whether the database is running, so a malformed config // aborts before the local database is recreated — the same pattern `db start`/`db push` use. yield* checkDbToml(fs, path, workdir); + if (backend.kind === "stack") { + const opened = yield* stackOpenReadyProject; + if (Option.isNone(opened)) return yield* Effect.fail(notRunning()); + yield* output.raw(`Resetting local database${toLogMessage(input.version)}\n`, "stderr"); + yield* opened.value.stack.resetDatabase.pipe( + Effect.catchTag("StackNotRunningError", () => Effect.fail(notRunning())), + Effect.mapError((cause) => resetFailed(`failed to reset local database: ${cause.message}`)), + ); + const catalog = yield* Effect.serviceOption(StackCatalogSetup); + if (Option.isNone(catalog)) return yield* resetFailed("stack catalog setup is unavailable"); + const stackConfig = yield* loadStackConfig(workdir).pipe( + Effect.mapError((cause) => resetFailed(cause.message)), + ); + const toml = yield* readDbToml(fs, path, workdir); + yield* catalog.value + .apply({ + target: { + kind: "live", + stack: opened.value.stack, + projectRoot: workdir, + config: stackConfig, + }, + overlay: { + webhooks: "config", + webhooksEnabled: toml.webhooksEnabled, + apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, + vault: toml.vault, + workdir, + }, + }) + .pipe(Effect.mapError((cause) => resetFailed(cause.message))); + const dbConn = yield* DbConnection; + const conn = yield* stackLocalDatabaseConn.pipe( + Effect.mapError((cause) => new ResetLocalDbNotRunningError({ message: cause.message })), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const session = yield* dbConn + .connect(conn, { isLocal: true, dnsResolver: "native" }) + .pipe( + Effect.mapError((cause) => + resetFailed(`failed to connect after reset: ${cause.message}`), + ), + ); + yield* migrateAndSeed(session, fs, path, workdir, input.version, { + migrationsEnabled: toml.migrationsEnabled, + seed: resolveResetSeedConfig(toml.seed, input.seedFlags, path), + experimental, + pgDeltaEnabled: toml.pgDelta.enabled, + schemaPaths: toml.schemaPaths, + localDatabaseWebhooksEnabled: toml.webhooksEnabled, + }).pipe(Effect.mapError((cause) => resetFailed(cause.message))); + }), + ); + const after = yield* opened.value.stack.status.pipe( + Effect.mapError((cause) => + resetFailed(`failed to inspect stack after reset: ${cause.message}`), + ), + ); + const storage = after.capabilities.find((capability) => capability.name === "storage"); + if (storage?.state === "ready") { + const context = yield* loadLocalProjectContext(workdir, (message) => resetFailed(message)); + yield* seedBucketsRun({ + projectRef: "", + emitSummary: false, + interactive: false, + yes, + resolvedConfig: { config: context.config, document: context.loaded?.document }, + projectEnvValues: projectEnv, + }).pipe( + Effect.catchTag("SeedConfigLoadError", (error) => + output.raw( + `${yellow("WARNING:")} skipped seeding storage buckets: ${error.message}\n`, + "stderr", + ), + ), + ); + } + const branch = Option.getOrElse(yield* detectGitBranch(workdir), () => "main"); + yield* output.raw( + `Finished ${aqua("supabase db reset")} on branch ${aqua(branch)}.\n`, + "stderr", + ); + return; + } + + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const runtimeInfo = yield* RuntimeInfo; + const networkIdFlag = yield* NetworkIdFlag; + // Threaded into `buildLocalDbContainerInputs`'s `setup.debug`, so a failed fresh-volume + // Realtime/Storage/Auth migrate job on the PG15 recreate path tees its own stderr. + const debug = yield* DebugFlag; + // Error if the local db container is down. const running = yield* isLocalDbRunning( spawner, @@ -95,11 +207,7 @@ export const resetLocalDatabase = Effect.fnUntraced(function* ( Option.getOrUndefined(cliSettings.projectId), ); if (!running) { - return yield* Effect.fail( - new ResetLocalDbNotRunningError({ - message: `${aqua("supabase start")} is not running.`, - }), - ); + return yield* Effect.fail(notRunning()); } // "Resetting local database…" then recreate + migrate + seed. yield* output.raw(`Resetting local database${toLogMessage(input.version)}\n`, "stderr"); diff --git a/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts b/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts index 8ae52abc9c..139e288d38 100644 --- a/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts +++ b/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts @@ -148,7 +148,7 @@ export interface ShadowCacheKeyInputs { * PG<=14 setup SQL is excluded because that major is cache-ineligible. */ let shadowBaselineEmbeddedDigestMemo: string | undefined; -const shadowBaselineEmbeddedDigest = (): string => +export const shadowBaselineEmbeddedDigest = (): string => (shadowBaselineEmbeddedDigestMemo ??= createHash("sha256") .update( [ @@ -171,7 +171,7 @@ const shadowBaselineEmbeddedDigest = (): string => .digest("hex")); /** JSON with recursively key-sorted objects, so `db.settings`' own property order cannot change the key. */ -function canonicalJson(value: unknown): string { +export function canonicalJson(value: unknown): string { if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; const entries = Object.entries(value) @@ -363,6 +363,8 @@ export interface ShadowBaselineRetentionOpts { readonly maxAgeMs?: number; /** Never evict this published tar, even if it is older than the TTL or over the cap. */ readonly retainFileName?: string; + /** Defaults to {@link isShadowBaselineTar}. */ + readonly isPublishedTar?: (fileName: string) => boolean; } /** @@ -377,8 +379,9 @@ export function shadowBaselineTarsToEvict( const keep = opts.keep ?? SHADOW_BASELINE_KEEP; const maxAgeMs = opts.maxAgeMs ?? SHADOW_BASELINE_MAX_AGE_MS; const retain = opts.retainFileName; + const isPublishedTar = opts.isPublishedTar ?? isShadowBaselineTar; const candidates = entries.filter( - (entry) => isShadowBaselineTar(entry.fileName) && entry.fileName !== retain, + (entry) => isPublishedTar(entry.fileName) && entry.fileName !== retain, ); const aged = new Set( candidates.filter((entry) => now - entry.mtimeMs > maxAgeMs).map((entry) => entry.fileName), @@ -469,7 +472,10 @@ const sweepShadowBaselineRetention = ( }); /** Refresh mtime on a warm hit so frequently used keys survive LRU/TTL. Best-effort. */ -const touchShadowBaselineTar = (fs: FileSystem.FileSystem, tarPath: string): Effect.Effect => +export const touchShadowBaselineTar = ( + fs: FileSystem.FileSystem, + tarPath: string, +): Effect.Effect => Effect.gen(function* () { const now = new Date(yield* Clock.currentTimeMillis); yield* fs.utimes(tarPath, now, now); diff --git a/apps/cli/src/command-internal/db-bootstrap/shadow-cache.unit.test.ts b/apps/cli/src/command-internal/db-bootstrap/shadow-cache.unit.test.ts index 313c8504df..27fc8f0a6f 100644 --- a/apps/cli/src/command-internal/db-bootstrap/shadow-cache.unit.test.ts +++ b/apps/cli/src/command-internal/db-bootstrap/shadow-cache.unit.test.ts @@ -412,4 +412,22 @@ describe("shadow baseline tar retention", () => { ), ).toEqual([]); }); + + it("never evicts retainFileName when using a custom published-tar matcher", () => { + const current = "stack-shadow-baseline-dddddddddddddddd.tar"; + const aged = now - SHADOW_BASELINE_MAX_AGE_MS - 1; + const isStack = (fileName: string) => + /^stack-shadow-baseline-[0-9a-f]{16}\.tar$/u.test(fileName); + const evicted = shadowBaselineTarsToEvict( + [ + { fileName: current, mtimeMs: aged }, + { fileName: "stack-shadow-baseline-aaaaaaaaaaaaaaaa.tar", mtimeMs: now - 1_000 }, + { fileName: "not-a-tar.json", mtimeMs: aged }, + ], + now, + { retainFileName: current, isPublishedTar: isStack }, + ); + expect(evicted).not.toContain(current); + expect(evicted).not.toContain("not-a-tar.json"); + }); }); diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index 767f04a909..1705ae097d 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { afterEach, beforeEach, describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, Layer, Option } from "effect"; +import { Effect, Exit, Layer, Option, Redacted, Stream } from "effect"; +import { CAPABILITY_NAMES, StackIdSchema, type EffectStack } from "@supabase/stack/effect"; import { mockAnalytics, @@ -22,11 +23,12 @@ import { } from "./global-flags.ts"; import { DebugLogger } from "./debug-logger.service.ts"; import { identityStitchLayer } from "./identity-stitch.ts"; -import { dbConfigLayer } from "./db-config.layer.ts"; +import { dbConfigLayer, dbConfigResolverLayer } from "./db-config.layer.ts"; import { DbConfigResolver } from "./db-config.service.ts"; import type { DbConfigFlags } from "./db-config.types.ts"; import { DbConnection, type DbSession, type PgConnInput } from "./db-connection.service.ts"; - +import { stackBackendLayer } from "./stack-backend.ts"; +import { StackApi } from "./stack-api.ts"; // `--local` / `--db-url` never touch the Management API stack, so the resolver // builds with simple ambient stubs. The `--linked` sub-flow (login-role, // pooler, unban, backoff) requires the real management runtime with a mocked @@ -46,6 +48,8 @@ function buildResolver( readonly projectHost?: string; readonly poolerHost?: string; readonly dbConnection?: Layer.Layer; + readonly stackApi?: Layer.Layer; + readonly stackBackend?: "legacy" | "stack"; } = {}, ) { const deps = Layer.mergeAll( @@ -76,7 +80,14 @@ function buildResolver( ), BunServices.layer, ); - return dbConfigLayer.pipe(Layer.provide(deps)); + const resolver = + opts.stackApi !== undefined + ? dbConfigResolverLayer.pipe(Layer.provide(opts.stackApi), Layer.provide(deps)) + : dbConfigLayer.pipe(Layer.provide(deps)); + return Layer.mergeAll( + resolver, + opts.stackBackend !== undefined ? stackBackendLayer(opts.stackBackend) : Layer.empty, + ); } function withWorkdir(toml?: string) { @@ -168,6 +179,77 @@ describe("dbConfigResolver (local + db-url)", () => { ); }); + it.effect("local mode: uses the stack credentials URL when the stack backend is on", () => { + const dir = withWorkdir(["[db]", "port = 55555", 'password = "hunter2"', ""].join("\n")); + const unused = () => Effect.die("unused"); + const unusedEffect = Effect.die("unused"); + const stackId = StackIdSchema.make("a".repeat(64)); + const stack: EffectStack = { + id: stackId, + status: Effect.succeed({ + id: stackId, + lifecycle: "running", + desiredLifecycle: "running", + runtime: { kind: "native" }, + endpoints: {}, + versions: {}, + capabilities: CAPABILITY_NAMES.map((name) => ({ + name, + activation: name === "database" ? "eager" : "lazy", + state: name === "database" ? "ready" : "dormant", + })), + artifacts: [], + }), + credentials: Effect.succeed({ + database: { + url: Redacted.make("postgresql://postgres:stack-secret@127.0.0.1:54329/postgres"), + password: Redacted.make("stack-secret"), + }, + api: { + publishableKey: "anon", + secretKey: Redacted.make("service"), + anonJwt: "anon", + serviceRoleJwt: Redacted.make("service"), + }, + }), + prepare: unused, + start: unused, + stop: unusedEffect, + destroy: unusedEffect, + resetDatabase: unusedEffect, + logs: unused, + followLogs: () => Stream.empty, + }; + const stackApi = Layer.succeed(StackApi, { + createStack: unused, + findStack: () => + Effect.succeed( + Option.some({ + id: stackId, + projectRoot: dir, + name: "default", + branchContext: "main", + runtime: { kind: "native" }, + desiredLifecycle: "running", + }), + ), + discoverStacks: unused, + openStack: () => Effect.succeed(stack), + inspectStack: unused, + }); + return resolve(dir, localFlags, { stackBackend: "stack", stackApi }).pipe( + Effect.tap((r) => + Effect.sync(() => { + expect(r.conn.host).toBe("127.0.0.1"); + expect(r.conn.port).toBe(54329); + expect(r.conn.password).toBe("stack-secret"); + expect(r.isLocal).toBe(true); + rmSync(dir, { recursive: true, force: true }); + }), + ), + ); + }); + it.effect("local mode: honors SUPABASE_SERVICES_HOSTNAME for the connection host", () => { process.env["SUPABASE_SERVICES_HOSTNAME"] = "host.docker.internal"; const dir = withWorkdir(); diff --git a/apps/cli/src/command-internal/db-config.layer.ts b/apps/cli/src/command-internal/db-config.layer.ts index 14f6b9ec45..8697a4641b 100644 --- a/apps/cli/src/command-internal/db-config.layer.ts +++ b/apps/cli/src/command-internal/db-config.layer.ts @@ -38,6 +38,9 @@ import type { DbConfigFlags } from "./db-config.types.ts"; import { DebugLogger } from "./debug-logger.service.ts"; import { getHostname } from "./hostname.ts"; import { mapHttpError } from "./http-errors.ts"; +import { currentStackBackend } from "./stack-backend.ts"; +import { StackApi, stackApiLayer } from "./stack-api.ts"; +import { stackLocalDatabaseConn } from "./stack-local-database.ts"; const DIRECT_PORT = 5432; const TCP_PROBE_TIMEOUT = Duration.seconds(5); @@ -375,10 +378,11 @@ export const resolveLinkedConn = Effect.fnUntraced(function* ( return poolerConn.value; }); -export const dbConfigLayer = Layer.effect( +export const dbConfigResolverLayer = Layer.effect( DbConfigResolver, Effect.gen(function* () { const cliSettings = yield* CommandSettings; + const stackApi = yield* StackApi; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const debug = yield* DebugLogger; @@ -561,6 +565,15 @@ export const dbConfigLayer = Layer.effect( const tomlValues = yield* readDbToml(fs, path, cliSettings.workdir, undefined, { resolveVaultSecrets, }); + const backend = yield* currentStackBackend; + if (backend.kind === "stack") { + // `resolve`'s R is `never`, so capture StackApi at layer build. + const conn = yield* stackLocalDatabaseConn.pipe( + Effect.provideService(CommandSettings, cliSettings), + Effect.provideService(StackApi, stackApi), + ); + return { conn, isLocal: true }; + } return { conn: { host: localHost, @@ -628,3 +641,5 @@ export const dbConfigLayer = Layer.effect( }); }), ); + +export const dbConfigLayer = dbConfigResolverLayer.pipe(Layer.provide(stackApiLayer)); diff --git a/apps/cli/src/command-internal/db-config.service.ts b/apps/cli/src/command-internal/db-config.service.ts index 18de77b180..040d98d280 100644 --- a/apps/cli/src/command-internal/db-config.service.ts +++ b/apps/cli/src/command-internal/db-config.service.ts @@ -10,6 +10,7 @@ import type { import type { ProfileLoadError } from "./profile-load.ts"; import type { ProjectRefReadError } from "./temp-paths.ts"; import type { DbConnectError } from "./db-connection.errors.ts"; +import type { LocalDbRunningError } from "./db-bootstrap/local-db-running.ts"; import type { DbConfigConnectTempRoleError, DbConfigIpv6Error, @@ -29,6 +30,7 @@ import type { DbConfigFlags, ResolvedDbConfig } from "./db-config.types.ts"; export type DbConfigError = | DbConfigParseUrlError | DbConfigLoadError + | LocalDbRunningError | ProjectRefNotLinkedError | InvalidProjectRefError // A hard linked-ref load surfaces a real `.temp/project-ref` read error instead of masking it diff --git a/apps/cli/src/command-internal/db-pull-run.ts b/apps/cli/src/command-internal/db-pull-run.ts index 89dcc90435..62f606c3ac 100644 --- a/apps/cli/src/command-internal/db-pull-run.ts +++ b/apps/cli/src/command-internal/db-pull-run.ts @@ -66,6 +66,9 @@ import { } from "../commands/db/shared/pgdelta-engine.service.ts"; import { type PgDeltaContext, isPgDeltaDebugEnabled, resolvePgDeltaProjectId } from "./pgdelta.ts"; import { prepareShadowSource } from "../commands/db/shared/shadow-source.ts"; +import { currentStackBackend } from "./stack-backend.ts"; +import { stackRejectNativeDockerDiffEngine } from "./stack-local-database.ts"; +import { stackPrepareShadowSource, stackWithShadowDatabase } from "./stack-shadow.ts"; import type { DbPullFlags } from "../commands/db/pull/pull.command.ts"; import { DbPullDumpError, @@ -366,12 +369,17 @@ export const runDbPull = Effect.fn("db.pull.run")(function* ( const usePgDeltaDiff = resolvePullDiffEngine({ engineFlagChanged: Option.isSome(flags.diffEngine), engine: Option.getOrElse(flags.diffEngine, () => "migra"), - pgDeltaDefault: shouldUsePgDelta({ - configEnabled: toml.pgDelta.enabled, - usePgDeltaFlag: false, - envEnabled: parseBoolEnv(toml.envLookup("SUPABASE_EXPERIMENTAL_PG_DELTA")), - }), + pgDeltaDefault: + (yield* currentStackBackend).kind === "stack" || + shouldUsePgDelta({ + configEnabled: toml.pgDelta.enabled, + usePgDeltaFlag: false, + envEnabled: parseBoolEnv(toml.envLookup("SUPABASE_EXPERIMENTAL_PG_DELTA")), + }), }); + if (Option.getOrElse(flags.diffEngine, () => "pg-delta") === "migra") { + yield* stackRejectNativeDockerDiffEngine; + } // Connectivity check, run before dialing. return yield* Effect.scoped( @@ -576,7 +584,10 @@ export const runDbPull = Effect.fn("db.pull.run")(function* ( const runShadowDiff = (targetEndpoint: PgDeltaDatabaseEndpoint) => Effect.gen(function* () { yield* output.raw("Creating shadow database...\n", "stderr"); - const resolvedPullShadowImage = yield* pullLocalInputs.resolvePostgresImage; + const stackBackend = (yield* currentStackBackend).kind === "stack"; + const resolvedPullShadowImage = stackBackend + ? "stack-ephemeral" + : yield* pullLocalInputs.resolvePostgresImage; const migrationMode: "legacy" | "pgdelta-next" = usePgDeltaDiff ? "pgdelta-next" : "legacy"; @@ -596,65 +607,70 @@ export const runDbPull = Effect.fn("db.pull.run")(function* ( schemaPaths: toml.schemaPathPatterns, pgDelta: toml.pgDelta, }; - // `withShadowDatabase` owns the interrupt-safe lifecycle and the cache seam. Each - // pooler-retry attempt still acquires and releases its own shadow; on the warm path - // every attempt restores a fresh container from the same cached snapshot. The key's - // webhooks policy must mirror what {@link prepareShadowSource} selects for this mode, - // or the two engines could restore each other's tars. - return yield* withShadowDatabase( - spawner, - shadowInput, - (handle) => - Effect.gen(function* () { - const shadow = yield* prepareShadowSource(spawner, handle, shadowInput); - const target = shadow.targetUrlOverride ?? targetEndpoint.ref; - yield* output.raw( - diffSchema.length > 0 - ? `Diffing schemas: ${diffSchema.join(",")}\n` - : "Diffing schemas...\n", - "stderr", - ); - if (usePgDeltaDiff) { - return yield* pgDeltaEngine.diffDatabase({ - context: ctx, - source: { - kind: "database", - ref: shadow.sourceUrl, - connectOptions: { isLocal: true, dnsResolver: "native" }, - }, - target: { - kind: "database", - ref: target, - ...(shadow.targetUrlOverride === undefined - ? { - ...(targetEndpoint.connection !== undefined - ? { connection: targetEndpoint.connection } - : {}), - connectOptions: targetEndpoint.connectOptions, - } - : { - connectOptions: { isLocal: true, dnsResolver }, - }), - }, - schema: diffSchema, - formatOptions, - debug: isPgDeltaDebugEnabled(), - strictCoverage: flags.strictCoverage, - }); - } - const sql = yield* diffMigra(ctx, { - source: shadow.sourceUrl, - target, + const runDiff = (shadow: { + readonly sourceUrl: string; + readonly targetUrlOverride: string | undefined; + }) => + Effect.gen(function* () { + const target = shadow.targetUrlOverride ?? targetEndpoint.ref; + yield* output.raw( + diffSchema.length > 0 + ? `Diffing schemas: ${diffSchema.join(",")}\n` + : "Diffing schemas...\n", + "stderr", + ); + if (usePgDeltaDiff) { + return yield* pgDeltaEngine.diffDatabase({ + context: ctx, + source: { + kind: "database", + ref: shadow.sourceUrl, + connectOptions: { isLocal: true, dnsResolver: "native" }, + }, + target: { + kind: "database", + ref: target, + ...(shadow.targetUrlOverride === undefined + ? { + ...(targetEndpoint.connection !== undefined + ? { connection: targetEndpoint.connection } + : {}), + connectOptions: targetEndpoint.connectOptions, + } + : { + connectOptions: { isLocal: true, dnsResolver }, + }), + }, schema: diffSchema, - connectOptions: - shadow.targetUrlOverride === undefined - ? targetEndpoint.connectOptions - : { isLocal: true, dnsResolver }, + formatOptions, + debug: isPgDeltaDebugEnabled(), + strictCoverage: flags.strictCoverage, }); - return { sql, files: undefined, debug: undefined }; - }), - { webhooks: migrationMode === "pgdelta-next" ? "config" : "enabled" }, - ); + } + const sql = yield* diffMigra(ctx, { + source: shadow.sourceUrl, + target, + schema: diffSchema, + connectOptions: + shadow.targetUrlOverride === undefined + ? targetEndpoint.connectOptions + : { isLocal: true, dnsResolver }, + }); + return { sql, files: undefined, debug: undefined }; + }); + return stackBackend + ? yield* stackWithShadowDatabase(shadowInput, (handle) => + stackPrepareShadowSource(handle, shadowInput).pipe(Effect.flatMap(runDiff)), + ) + : // `withShadowDatabase` owns the interrupt-safe lifecycle and the cache seam. + // Webhooks policy must mirror {@link prepareShadowSource} for this mode. + yield* withShadowDatabase( + spawner, + shadowInput, + (handle) => + prepareShadowSource(spawner, handle, shadowInput).pipe(Effect.flatMap(runDiff)), + { webhooks: migrationMode === "pgdelta-next" ? "config" : "enabled" }, + ); }); const diffOutcome = yield* withPoolerFallback(targetEndpoint, runShadowDiff); diff --git a/apps/cli/src/command-internal/pg-dump.run.ts b/apps/cli/src/command-internal/pg-dump.run.ts index f8fbf9f5de..9518d6655e 100644 --- a/apps/cli/src/command-internal/pg-dump.run.ts +++ b/apps/cli/src/command-internal/pg-dump.run.ts @@ -5,6 +5,8 @@ import { viperEnvStringWithProjectFallback } from "./viper-env.ts"; import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; import { getRegistryImageUrl } from "./docker-registry.ts"; import { DockerRun } from "./docker-run.service.ts"; +import { currentStackBackend } from "./stack-backend.ts"; +import { requireHostPostgresClient, streamHostCommand } from "./postgres-client.run.ts"; /** * Runs a pg_dump/pg_dumpall bash script in a one-shot container, streaming stdout @@ -31,6 +33,12 @@ export const streamPgDump = Effect.fnUntraced(function* (params: { * (or `{}`) by callers that haven't loaded a project env map. */ readonly projectEnvValues?: Readonly>; + /** + * Stack dumps always talk to published credentials. Ignore compose + * `SUPABASE_NETWORK_ID` so the tool container never joins `supabase_network_*`. + * An explicit `--network-id` still wins. + */ + readonly forceHostNetwork?: boolean; }) { const docker = yield* DockerRun; const runtimeInfo = yield* RuntimeInfo; @@ -40,10 +48,9 @@ export const streamPgDump = Effect.fnUntraced(function* (params: { // precedence order. The generated `supabase_network_*` fallback used elsewhere never // applies here, since this path always sets a NetworkMode. const networkId = Option.getOrUndefined(networkIdFlag); - const envNetworkId = viperEnvStringWithProjectFallback( - "SUPABASE_NETWORK_ID", - params.projectEnvValues ?? {}, - ); + const envNetworkId = params.forceHostNetwork + ? "" + : viperEnvStringWithProjectFallback("SUPABASE_NETWORK_ID", params.projectEnvValues ?? {}); const network = networkId !== undefined && networkId.length > 0 ? { _tag: "named" as const, name: networkId } @@ -66,3 +73,42 @@ export const streamPgDump = Effect.fnUntraced(function* (params: { { onStdout: params.onStdout, teeStderr: true }, ); }); + +export type PgDumpClient = + | { readonly kind: "container" } + | { + readonly kind: "host"; + readonly command: "pg_dump" | "pg_dumpall"; + readonly expectedMajor: number; + }; + +export const pgDumpClientExitMessage = (client: PgDumpClient, exitCode: number): string => + client.kind === "host" + ? `error running ${client.command}: exit ${exitCode}` + : `error running container: exit ${exitCode}`; + +/** Container dump, or PATH `pg_dump`/`pg_dumpall` when the stack engine is native. */ +export const streamPgDumpWithClient = Effect.fnUntraced(function* (params: { + readonly image: string; + readonly script: string; + readonly env: Readonly>; + readonly onStdout: (chunk: Uint8Array) => Effect.Effect; + readonly projectEnvValues?: Readonly>; + readonly client: PgDumpClient; +}) { + if (params.client.kind === "host") { + yield* requireHostPostgresClient(params.client.command, params.client.expectedMajor); + return yield* streamHostCommand({ + command: "bash", + args: ["-c", params.script, "--"], + env: params.env, + onStdout: params.onStdout, + teeStderr: true, + }); + } + const backend = yield* currentStackBackend; + return yield* streamPgDump({ + ...params, + forceHostNetwork: backend.kind === "stack", + }); +}); diff --git a/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts b/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts index 04fc9f26ad..a23b0be224 100644 --- a/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts +++ b/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts @@ -14,6 +14,9 @@ import { pgDeltaNextAdapterLayer } from "../commands/db/shared/pgdelta-next-adap import { pgDeltaNextShadowLayer } from "../commands/db/shared/pgdelta-next-shadow.layer.ts"; import { declarativeSeamLayer } from "../commands/db/shared/pgdelta.seam.layer.ts"; import { localDockerEngineLayer } from "./db-bootstrap/local-db-running.ts"; +import { stackApiLayer } from "./stack-api.ts"; +import { ephemeralPostgresLayer } from "./stack-shadow.ts"; +import { stackCatalogSetupLayer } from "./stack-catalog-setup.ts"; /** The in-process pg-delta engine — the only implementation. */ const pgDeltaEngineLayer = pgDeltaNextEngineLayer; @@ -57,6 +60,7 @@ const nextShadow = pgDeltaNextShadowLayer.pipe( Layer.provide(dockerRunLayer), Layer.provide(dbConnectionLayer), Layer.provide(httpClient), + Layer.provide(pgDeltaCommandSettingsRuntimeLayer), ); const engine = pgDeltaEngineLayer.pipe( Layer.provide(pgDeltaCommandSettingsRuntimeLayer), @@ -77,4 +81,7 @@ export const pgDeltaCommandRuntimeLayer = Layer.mergeAll( pgDeltaCommandSettingsRuntimeLayer, // Exposed for handlers' own direct `isLocalDbRunning` calls (`db diff --use-pgadmin`). localDockerEngine, + stackApiLayer, + ephemeralPostgresLayer, + stackCatalogSetupLayer, ); diff --git a/apps/cli/src/command-internal/postgres-client.run.ts b/apps/cli/src/command-internal/postgres-client.run.ts new file mode 100644 index 0000000000..81394b4eef --- /dev/null +++ b/apps/cli/src/command-internal/postgres-client.run.ts @@ -0,0 +1,189 @@ +import { Data, Effect, Option, Result, Stream } from "effect"; +import * as ChildProcess from "effect/unstable/process/ChildProcess"; +import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; + +import { ProcessControl } from "../shared/runtime/process-control.service.ts"; +import { + actionability, + type CliErrorActionabilityDeclaration, + ErrorActionabilityId, +} from "../shared/telemetry/error-actionability.ts"; +import { collectText } from "./container-cli.ts"; +import type { PgConnInput } from "./db-connection.service.ts"; + +const POSTGRES_CLIENT_MAJOR = /\(PostgreSQL\)\s+(\d+)/; +const HOST_CLIENT_SUGGESTION = + "Install matching PostgreSQL client tools on PATH, or start the stack with --runtime docker."; + +export const parsePostgresClientMajor = (text: string): number | undefined => { + const match = POSTGRES_CLIENT_MAJOR.exec(text); + if (match?.[1] === undefined) return undefined; + const major = Number(match[1]); + return Number.isInteger(major) ? major : undefined; +}; + +/** `pg_prove` has no Postgres major; any matching `psql` or `pg_dump` on PATH is enough. */ +export const matchingHostPostgresClient = ( + dumpMajor: number | undefined, + psqlMajor: number | undefined, + expected: number, +): + | { readonly kind: "match" } + | { + readonly kind: "mismatch"; + readonly command: "pg_dump" | "psql"; + readonly actual: number | undefined; + } => { + if (dumpMajor === expected || psqlMajor === expected) return { kind: "match" }; + if (psqlMajor !== undefined) return { kind: "mismatch", command: "psql", actual: psqlMajor }; + return { kind: "mismatch", command: "pg_dump", actual: dumpMajor }; +}; + +export class HostPostgresClientError extends Data.TaggedError("HostPostgresClientError")<{ + readonly message: string; + readonly suggestion?: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.provideFlags; + } +} + +const missingClient = (command: string) => + new HostPostgresClientError({ + message: `${command} was not found on PATH.`, + suggestion: HOST_CLIENT_SUGGESTION, + }); + +const majorMismatch = (command: string, actual: number | undefined, expected: number) => + new HostPostgresClientError({ + message: + actual === undefined + ? `${command} did not report a PostgreSQL major version.` + : `${command} major version ${actual} does not match stack Postgres ${expected}.`, + suggestion: HOST_CLIENT_SUGGESTION, + }); + +const hostClientVersion = (command: string) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner; + const handle = yield* spawner.spawn( + ChildProcess.make(command, ["--version"], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), + ); + const [exitCode, stdout, stderr] = yield* Effect.all( + [ + handle.exitCode.pipe( + Effect.map(Number), + Effect.mapError(() => missingClient(command)), + ), + collectText(handle.stdout.pipe(Stream.mapError(() => missingClient(command)))), + collectText(handle.stderr.pipe(Stream.mapError(() => missingClient(command)))), + ], + { concurrency: "unbounded" }, + ); + return { exitCode, output: `${stdout}\n${stderr}` }; + }), + ).pipe( + Effect.mapError(() => missingClient(command)), + Effect.flatMap((result) => + result.exitCode === 0 ? Effect.succeed(result.output) : Effect.fail(missingClient(command)), + ), + ); + +/** Require a PATH client whose `--version` major matches the stack Postgres. */ +export const requireHostPostgresClient = ( + command: string, + expectedMajor: number, +): Effect.Effect => + Effect.gen(function* () { + const output = yield* hostClientVersion(command); + const major = parsePostgresClientMajor(output); + if (major !== expectedMajor) return yield* majorMismatch(command, major, expectedMajor); + }); + +/** + * `pg_prove --version` has no Postgres major. Require `pg_prove` on PATH and a + * matching `pg_dump` or `psql` major. + */ +export const requireHostPgProve = ( + expectedMajor: number, +): Effect.Effect => + Effect.gen(function* () { + yield* hostClientVersion("pg_prove"); + const dump = yield* hostClientVersion("pg_dump").pipe(Effect.result); + const psql = yield* hostClientVersion("psql").pipe(Effect.result); + const dumpMajor = Result.isSuccess(dump) ? parsePostgresClientMajor(dump.success) : undefined; + const psqlMajor = Result.isSuccess(psql) ? parsePostgresClientMajor(psql.success) : undefined; + const matched = matchingHostPostgresClient(dumpMajor, psqlMajor, expectedMajor); + if (matched.kind === "match") return; + return yield* majorMismatch(matched.command, matched.actual, expectedMajor); + }); + +/** Stream a host process stdout like `streamPgDump`, teeing stderr when requested. */ +export const streamHostCommand = Effect.fnUntraced(function* (params: { + readonly command: string; + readonly args: ReadonlyArray; + readonly env: Readonly>; + readonly cwd?: string; + readonly onStdout: (chunk: Uint8Array) => Effect.Effect; + readonly teeStderr?: boolean; + readonly captureStderr?: boolean; +}) { + const spawner = yield* ChildProcessSpawner; + const processControl = yield* Effect.serviceOption(ProcessControl); + const teeStderr = params.teeStderr ?? false; + const captureStderr = params.captureStderr ?? true; + return yield* Effect.scoped( + Effect.gen(function* () { + if (Option.isSome(processControl)) { + yield* processControl.value.holdSignals(["SIGINT", "SIGTERM", "SIGHUP"]); + } + const handle = yield* spawner + .spawn( + ChildProcess.make(params.command, [...params.args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + cwd: params.cwd, + env: params.env, + extendEnv: true, + }), + ) + .pipe(Effect.mapError(() => missingClient(params.command))); + const stderrChunks: Array = []; + yield* Effect.all( + [ + Stream.runForEach(handle.stdout, params.onStdout), + Stream.runForEach(handle.stderr, (chunk) => + Effect.sync(() => { + if (captureStderr) stderrChunks.push(chunk); + if (teeStderr) globalThis.process.stderr.write(chunk); + }), + ), + ], + { concurrency: "unbounded" }, + ); + const exitCode = yield* handle.exitCode.pipe(Effect.map(Number)); + return { exitCode, stderr: new TextDecoder().decode(Buffer.concat(stderrChunks)) }; + }), + ); +}); + +/** Native-engine dumps talk to loopback; container tools may need Docker Desktop's host alias. */ +export const rewriteDumpHostForToolContainer = ( + host: string, + opts: { readonly platform: string; readonly usesHostNetwork: boolean }, +): string => { + if (host !== "127.0.0.1" && host !== "localhost") return host; + if (opts.platform !== "linux" || !opts.usesHostNetwork) return "host.docker.internal"; + return host; +}; + +export const dumpConnForHostClient = (conn: PgConnInput): PgConnInput => ({ + ...conn, + host: "127.0.0.1", +}); diff --git a/apps/cli/src/command-internal/postgres-client.run.unit.test.ts b/apps/cli/src/command-internal/postgres-client.run.unit.test.ts new file mode 100644 index 0000000000..b786ae51b9 --- /dev/null +++ b/apps/cli/src/command-internal/postgres-client.run.unit.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { matchingHostPostgresClient, parsePostgresClientMajor } from "./postgres-client.run.ts"; + +describe("parsePostgresClientMajor", () => { + it("reads the PostgreSQL major from client --version output", () => { + expect(parsePostgresClientMajor("pg_dump (PostgreSQL) 17.4")).toBe(17); + expect(parsePostgresClientMajor("psql (PostgreSQL) 15.12")).toBe(15); + expect(parsePostgresClientMajor("pg_dumpall (PostgreSQL) 16.1")).toBe(16); + }); + + it("returns undefined when the version line has no PostgreSQL major", () => { + expect(parsePostgresClientMajor("pg_prove version 3.36")).toBeUndefined(); + expect(parsePostgresClientMajor("")).toBeUndefined(); + }); +}); + +describe("matchingHostPostgresClient", () => { + it("accepts a matching psql when pg_dump reports another major", () => { + expect(matchingHostPostgresClient(16, 17, 17)).toEqual({ kind: "match" }); + expect(matchingHostPostgresClient(17, 16, 17)).toEqual({ kind: "match" }); + }); + + it("fails only when neither client matches", () => { + expect(matchingHostPostgresClient(16, undefined, 17)).toEqual({ + kind: "mismatch", + command: "pg_dump", + actual: 16, + }); + expect(matchingHostPostgresClient(undefined, 15, 17)).toEqual({ + kind: "mismatch", + command: "psql", + actual: 15, + }); + expect(matchingHostPostgresClient(undefined, undefined, 17)).toEqual({ + kind: "mismatch", + command: "pg_dump", + actual: undefined, + }); + }); +}); diff --git a/apps/cli/src/command-internal/stack-api.ts b/apps/cli/src/command-internal/stack-api.ts new file mode 100644 index 0000000000..09249ac4a3 --- /dev/null +++ b/apps/cli/src/command-internal/stack-api.ts @@ -0,0 +1,70 @@ +import { Context, Crypto, Effect, FileSystem, Layer, Path } from "effect"; +import { + createStack, + discoverStacks, + findStack, + inspectStack, + openStack, + type StackDiscoveryResult, +} from "@supabase/stack/effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; + +export class StackApi extends Context.Service< + StackApi, + { + readonly findStack: ( + ...args: Parameters + ) => Effect.Effect< + Effect.Success>, + Effect.Error> + >; + readonly createStack: ( + ...args: Parameters + ) => Effect.Effect< + Effect.Success>, + Effect.Error> + >; + readonly openStack: ( + ...args: Parameters + ) => Effect.Effect< + Effect.Success>, + Effect.Error> + >; + readonly inspectStack: ( + ...args: Parameters + ) => Effect.Effect< + Effect.Success>, + Effect.Error> + >; + readonly discoverStacks: ( + ...args: Parameters + ) => Effect.Effect>>; + } +>()("supabase/experimental-stack/StackApi") {} + +export const stackApiLayer = Layer.effect( + StackApi, + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const childProcess = yield* ChildProcessSpawner.ChildProcessSpawner; + const provideServices = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcess), + ); + return { + findStack: (...args: Parameters) => provideServices(findStack(...args)), + createStack: (...args: Parameters) => + provideServices(createStack(...args)), + openStack: (...args: Parameters) => provideServices(openStack(...args)), + inspectStack: (...args: Parameters) => + provideServices(inspectStack(...args)), + discoverStacks: (...args: Parameters) => + provideServices(discoverStacks(...args)), + }; + }), +); diff --git a/apps/cli/src/commands/experimental/stack/stack-backend.ts b/apps/cli/src/command-internal/stack-backend.ts similarity index 79% rename from apps/cli/src/commands/experimental/stack/stack-backend.ts rename to apps/cli/src/command-internal/stack-backend.ts index 82a7dd2beb..d7b789721b 100644 --- a/apps/cli/src/commands/experimental/stack/stack-backend.ts +++ b/apps/cli/src/command-internal/stack-backend.ts @@ -1,17 +1,20 @@ import { CliConfigSchema } from "@supabase/config/effect"; -import { Data, Effect, FileSystem, Option, Path, Schema } from "effect"; +import { Context, Data, Effect, FileSystem, Layer, Option, Path, Schema } from "effect"; import * as SmolToml from "smol-toml"; -import { resolveWorkdir } from "../../../config/command-settings.layer.ts"; -import { resolveExperimentalFeature } from "../../../command-internal/experimental-feature.ts"; -import { extractCommandPath, hasRootVersionFlag, rootFlagTokens } from "../../../shared/cli/run.ts"; +import { resolveWorkdir } from "../config/command-settings.layer.ts"; +import { resolveExperimentalFeature } from "./experimental-feature.ts"; +import { extractCommandPath, hasRootVersionFlag, rootFlagTokens } from "../shared/cli/run.ts"; import { actionability, type CliErrorActionabilityDeclaration, ErrorActionabilityId, -} from "../../../shared/telemetry/error-actionability.ts"; +} from "../shared/telemetry/error-actionability.ts"; export type StackBackend = "legacy" | "stack"; +/** Commands that consult experimental.stack for local database and shadow routing. */ +const STACK_BACKEND_COMMANDS = new Set(["start", "stop", "db", "migration", "test"]); + export class StackRoutingError extends Data.TaggedError("StackRoutingError")<{ readonly message: string; readonly cause?: unknown; @@ -25,6 +28,21 @@ export class StackRoutingError extends Data.TaggedError("StackRoutingError")<{ } } +/** In-process backend selected before parse; handlers must not re-read argv. */ +export class StackBackendContext extends Context.Service< + StackBackendContext, + { readonly kind: StackBackend } +>()("supabase/stack/Backend") {} + +export const stackBackendLayer = (kind: StackBackend) => + Layer.succeed(StackBackendContext, { kind }); + +/** Handlers default to legacy when tests omit the root-provided backend service. */ +export const currentStackBackend: Effect.Effect<{ readonly kind: StackBackend }, never, never> = + Effect.serviceOption(StackBackendContext).pipe( + Effect.map((value) => Option.getOrElse(value, () => ({ kind: "legacy" as const }))), + ); + const stackRoutingSchema = Schema.Struct({ experimental: Schema.optionalKey( Schema.Struct({ stack: CliConfigSchema.fields.experimental.to.fields.stack }), @@ -92,7 +110,7 @@ export const resolveStackBackend = (input: { // The explicit namespace is always backed by the stack runtime and does // not need a project config or environment lookup to select it. if (command === "stack") return "stack"; - if (command !== "start" && command !== "stop") return "legacy"; + if (command === undefined || !STACK_BACKEND_COMMANDS.has(command)) return "legacy"; const configValue = Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/cli/src/command-internal/stack-catalog-setup.ts b/apps/cli/src/command-internal/stack-catalog-setup.ts new file mode 100644 index 0000000000..702aaf0ad3 --- /dev/null +++ b/apps/cli/src/command-internal/stack-catalog-setup.ts @@ -0,0 +1,219 @@ +import { Context, Crypto, Data, Effect, FileSystem, Layer, Path, Redacted } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { + schemaInit, + type EffectStack, + type SchemaInitCapabilityName, + type SchemaInitTarget, + type StackConfig, + type StackRuntime, +} from "@supabase/stack/effect"; +import { Output } from "../shared/output/output.service.ts"; +import { parseConnectionString } from "./db-config.parse.ts"; +import { DbConnection } from "./db-connection.service.ts"; +import { dbConnectionLayer } from "./db-connection.layer.ts"; +import { + applyDatabaseOverlay, + type ApplyDatabaseOverlayInput, + type SetupDatabaseOptions, +} from "./db-bootstrap/db-setup.ts"; +import type { VaultSecret } from "./vault.ts"; + +const PLATFORM_TRIO = [ + "auth", + "storage", + "realtime", +] as const satisfies ReadonlyArray; +const OPTIONAL_CAPS = [ + "analytics", + "pooler", +] as const satisfies ReadonlyArray; + +export class StackCatalogSetupError extends Data.TaggedError("StackCatalogSetupError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +interface StackCatalogOverlay { + readonly webhooks?: SetupDatabaseOptions["webhooks"]; + readonly webhooksEnabled: boolean; + readonly apiAutoExposeNewTables: ApplyDatabaseOverlayInput["apiAutoExposeNewTables"]; + readonly vault: ReadonlyArray; + readonly workdir: string; + readonly announceRoles?: boolean; +} + +interface LiveStackCatalogInput { + readonly kind: "live"; + readonly stack: EffectStack; + readonly projectRoot: string; + readonly config: StackConfig; +} + +interface EphemeralStackCatalogInput { + readonly kind: "ephemeral"; + readonly projectRoot: string; + readonly runtime: StackRuntime; + readonly config: StackConfig; + readonly databaseUrl: string; + readonly databasePassword: Redacted.Redacted; + readonly jwtSecret?: Redacted.Redacted; + readonly networkId?: string; +} + +export interface StackCatalogSetupInput { + readonly target: LiveStackCatalogInput | EphemeralStackCatalogInput; + readonly overlay: StackCatalogOverlay; +} + +const capabilityEnabled = (config: StackConfig, name: SchemaInitCapabilityName): boolean => { + const cap = config.capabilities?.[name]; + return cap === undefined || cap.enabled !== false; +}; + +const jwtSecretFromConfig = (config: StackConfig): Redacted.Redacted | undefined => { + const signing = config.security?.jwt?.signing; + return signing?.kind === "symmetric" ? signing.secret : undefined; +}; + +const catalogError = (error: { readonly message: string }): StackCatalogSetupError => + new StackCatalogSetupError({ message: error.message, cause: error }); + +const runSchemaInit = (names: ReadonlyArray, target: SchemaInitTarget) => + names.length === 0 ? Effect.void : schemaInit(names, target).pipe(Effect.mapError(catalogError)); + +const targetConnection = (target: LiveStackCatalogInput | EphemeralStackCatalogInput) => + target.kind === "ephemeral" + ? Effect.succeed({ + databaseUrl: target.databaseUrl, + databasePassword: target.databasePassword, + jwtSecret: target.jwtSecret, + runtime: target.runtime, + }) + : Effect.gen(function* () { + const credentials = yield* target.stack.credentials; + const status = yield* target.stack.status; + return { + databaseUrl: Redacted.value(credentials.database.url), + databasePassword: credentials.database.password, + jwtSecret: jwtSecretFromConfig(target.config), + runtime: status.runtime, + }; + }).pipe(Effect.mapError(catalogError)); + +const applyCatalog = (input: StackCatalogSetupInput) => + Effect.gen(function* () { + const output = yield* Output; + const dbConn = yield* DbConnection; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const connection = yield* targetConnection(input.target); + const schemaTarget: SchemaInitTarget = + input.target.kind === "live" + ? { + kind: "live", + stackId: input.target.stack.id, + projectRoot: input.target.projectRoot, + runtime: connection.runtime, + config: input.target.config, + databaseUrl: connection.databaseUrl, + secrets: { + databasePassword: connection.databasePassword, + ...(connection.jwtSecret === undefined ? {} : { jwtSecret: connection.jwtSecret }), + }, + } + : { + kind: "ephemeral", + projectRoot: input.target.projectRoot, + runtime: connection.runtime, + config: input.target.config, + databaseUrl: connection.databaseUrl, + secrets: { + databasePassword: connection.databasePassword, + ...(connection.jwtSecret === undefined ? {} : { jwtSecret: connection.jwtSecret }), + }, + ...(input.target.networkId === undefined ? {} : { networkId: input.target.networkId }), + }; + const config = input.target.config; + const failClosed = PLATFORM_TRIO.filter((name) => capabilityEnabled(config, name)); + yield* runSchemaInit(failClosed, schemaTarget); + if (input.target.kind === "live") { + const optional = OPTIONAL_CAPS.filter((name) => capabilityEnabled(config, name)); + yield* Effect.forEach( + optional, + (name) => + schemaInit([name], schemaTarget).pipe( + Effect.catchTag("RequiresActivatedProcessError", (error) => + output.raw( + `WARNING: skipped ${error.capability} schema init: ${error.message}\n`, + "stderr", + ), + ), + Effect.mapError(catalogError), + ), + { discard: true }, + ); + } + const conn = parseConnectionString(connection.databaseUrl); + if (conn === undefined) { + return yield* new StackCatalogSetupError({ + message: "failed to parse database URL for catalog overlay", + }); + } + yield* Effect.scoped( + Effect.gen(function* () { + const session = yield* dbConn.connect(conn, { isLocal: true, dnsResolver: "native" }); + yield* applyDatabaseOverlay(session, fs, path, input.overlay.workdir, { + webhooksEnabled: input.overlay.webhooksEnabled, + apiAutoExposeNewTables: input.overlay.apiAutoExposeNewTables, + vault: input.overlay.vault, + webhooks: input.overlay.webhooks, + announceRoles: input.overlay.announceRoles, + }); + }).pipe(Effect.mapError(catalogError)), + ); + }); + +export class StackCatalogSetup extends Context.Service< + StackCatalogSetup, + { + readonly apply: ( + input: StackCatalogSetupInput, + ) => Effect.Effect; + } +>()("supabase/cli/StackCatalogSetup") {} + +type CatalogApplyServices = + | DbConnection + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner; + +export const stackCatalogSetupLayer = Layer.effect( + StackCatalogSetup, + Effect.gen(function* () { + const context = yield* Effect.context(); + return { + apply: (input: StackCatalogSetupInput) => + applyCatalog(input).pipe(Effect.provideContext(context), Effect.asVoid), + }; + }), +).pipe(Layer.provide(dbConnectionLayer)); + +export const noopStackCatalogSetupLayer = Layer.succeed(StackCatalogSetup, { + apply: () => Effect.void, +}); + +export const recordingStackCatalogSetup = (record: (input: StackCatalogSetupInput) => A) => { + const applied: Array = []; + return { + applied, + layer: Layer.succeed(StackCatalogSetup, { + apply: (input) => + Effect.sync(() => { + applied.push(record(input)); + }), + }), + }; +}; diff --git a/apps/cli/src/commands/experimental/stack/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts similarity index 99% rename from apps/cli/src/commands/experimental/stack/stack-config.ts rename to apps/cli/src/command-internal/stack-config.ts index 0f49b450c2..38f073760d 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -2,8 +2,8 @@ import { type CliConfig, validateCliConfig } from "@supabase/config/effect"; import { Effect, Data, FileSystem, Option, Path, Redacted, Schema, SchemaIssue } from "effect"; import { StackConfigSchema, type StackConfig } from "@supabase/stack/effect"; -import { loadLocalProjectContext } from "../../../command-internal/local-project-context.ts"; -import { parseDotEnv } from "../../../command-internal/dotenv.ts"; +import { loadLocalProjectContext } from "./local-project-context.ts"; +import { parseDotEnv } from "./dotenv.ts"; import { envOverride, envOverrideApiMaxRows, @@ -33,17 +33,13 @@ import { resolveGotrueSessions, resolveGotrueWeb3, strToArr, -} from "../../../command-internal/local-config-values.ts"; -import { - collectDotenvPrivateKeys, - decryptSecret, - isEncryptedSecret, -} from "../../../command-internal/vault-decrypt.ts"; +} from "./local-config-values.ts"; +import { collectDotenvPrivateKeys, decryptSecret, isEncryptedSecret } from "./vault-decrypt.ts"; import { actionability, type CliErrorActionabilityDeclaration, ErrorActionabilityId, -} from "../../../shared/telemetry/error-actionability.ts"; +} from "../shared/telemetry/error-actionability.ts"; /** A config error suitable for a stack command's user-facing boundary. */ export class StackConfigError extends Data.TaggedError("StackConfigError")<{ diff --git a/apps/cli/src/command-internal/stack-local-database.integration.test.ts b/apps/cli/src/command-internal/stack-local-database.integration.test.ts new file mode 100644 index 0000000000..d829be0591 --- /dev/null +++ b/apps/cli/src/command-internal/stack-local-database.integration.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Layer, Option, Redacted, Stream } from "effect"; +import { CAPABILITY_NAMES, StackIdSchema, type EffectStack } from "@supabase/stack/effect"; +import { mockCommandSettings, useTempWorkdir } from "../../tests/helpers/command-mocks.ts"; +import { stackBackendLayer } from "./stack-backend.ts"; +import { stackLocalDatabaseUrl } from "./stack-local-database.ts"; +import { StackApi } from "./stack-api.ts"; +const tmp = useTempWorkdir("stack-local-db-"); +const STACK_ID = StackIdSchema.make("a".repeat(64)); + +const unused = () => Effect.die("unused"); +const unusedEffect = Effect.die("unused"); + +const stack: EffectStack = { + id: STACK_ID, + status: Effect.succeed({ + id: STACK_ID, + lifecycle: "running", + desiredLifecycle: "running", + runtime: { kind: "native" }, + endpoints: {}, + versions: {}, + capabilities: CAPABILITY_NAMES.map((name) => ({ + name, + activation: name === "database" ? "eager" : "lazy", + state: name === "database" ? "ready" : "dormant", + })), + artifacts: [], + }), + credentials: Effect.succeed({ + database: { + url: Redacted.make("postgresql://postgres:secret@127.0.0.1:54329/postgres"), + password: Redacted.make("secret"), + }, + api: { + publishableKey: "anon", + secretKey: Redacted.make("service"), + anonJwt: "anon", + serviceRoleJwt: Redacted.make("service"), + }, + }), + prepare: unused, + start: unused, + stop: unusedEffect, + destroy: unusedEffect, + resetDatabase: unusedEffect, + logs: unused, + followLogs: () => Stream.empty, +}; + +describe("stackLocalDatabaseUrl", () => { + it.effect("returns the project stack database URL when the database is ready", () => { + const api = Layer.succeed(StackApi, { + createStack: unused, + findStack: () => + Effect.succeed( + Option.some({ + id: STACK_ID, + projectRoot: tmp.current, + name: "default", + branchContext: "main", + runtime: { kind: "native" }, + desiredLifecycle: "running", + }), + ), + discoverStacks: unused, + openStack: () => Effect.succeed(stack), + inspectStack: unused, + }); + return Effect.gen(function* () { + expect(yield* stackLocalDatabaseUrl).toBe( + "postgresql://postgres:secret@127.0.0.1:54329/postgres", + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + mockCommandSettings({ workdir: tmp.current }), + stackBackendLayer("stack"), + api, + ), + ), + ); + }); +}); diff --git a/apps/cli/src/command-internal/stack-local-database.ts b/apps/cli/src/command-internal/stack-local-database.ts new file mode 100644 index 0000000000..0331ee83f0 --- /dev/null +++ b/apps/cli/src/command-internal/stack-local-database.ts @@ -0,0 +1,248 @@ +import { Data, Effect, FileSystem, Option, Path, Redacted } from "effect"; +import { Output } from "../shared/output/output.service.ts"; +import { + actionability, + type CliErrorActionabilityDeclaration, + ErrorActionabilityId, +} from "../shared/telemetry/error-actionability.ts"; +import { + CAPABILITY_NAMES, + excludeStackCapabilities, + type EffectStack, + type StackConfig, + type StackRuntime, +} from "@supabase/stack/effect"; +import { parseConnectionString } from "./db-config.parse.ts"; +import type { PgConnInput } from "./db-connection.service.ts"; +import { CommandSettings } from "../config/command-settings.service.ts"; +import { LocalDbRunningError } from "./db-bootstrap/local-db-running.ts"; +import { currentStackBackend } from "./stack-backend.ts"; +import { StackApi } from "./stack-api.ts"; +import { loadStackConfig } from "./stack-config.ts"; +import { readDbToml } from "./db-config.toml-read.ts"; +import { StackCatalogSetup } from "./stack-catalog-setup.ts"; + +const notRunning = (message = "supabase start is not running.") => + new LocalDbRunningError({ message }); + +const startFailed = (cause: { readonly message: string }) => + new LocalDbRunningError({ + message: `failed to start local database: ${cause.message}`, + }); + +/** Capabilities `db start` and `stack start --exclude` can leave disabled. */ +export const STACK_START_EXCLUDABLE_CAPABILITIES = CAPABILITY_NAMES.filter( + (name) => name !== "database", +); + +const postgresOnlyStackStartConfig = (config: StackConfig): StackConfig => + excludeStackCapabilities(config, STACK_START_EXCLUDABLE_CAPABILITIES); + +const databaseReady = (stack: EffectStack) => + Effect.gen(function* () { + const status = yield* stack.status.pipe(Effect.mapError((cause) => notRunning(cause.message))); + const database = status.capabilities.find((capability) => capability.name === "database"); + if (status.lifecycle !== "running" || database?.state !== "ready") return Option.none(); + return Option.some({ stack, runtime: status.runtime }); + }); + +const openProjectStack = Effect.gen(function* () { + const api = yield* Effect.serviceOption(StackApi); + if (Option.isNone(api)) return Option.none(); + const cliSettings = yield* CommandSettings; + const descriptor = yield* api.value + .findStack({ projectRoot: cliSettings.workdir }) + .pipe(Effect.mapError((cause) => notRunning(cause.message))); + if (Option.isNone(descriptor)) return Option.none(); + const stack = yield* api.value + .openStack(descriptor.value.id) + .pipe(Effect.mapError((cause) => notRunning(cause.message))); + return yield* databaseReady(stack); +}); + +/** Ready project stack, or none when the stack is missing or the database is not ready. */ +export const stackOpenReadyProject = openProjectStack; + +export const stackProjectRuntime: Effect.Effect = + Effect.gen(function* () { + const api = yield* Effect.serviceOption(StackApi); + if (Option.isNone(api)) return undefined; + const cliSettings = yield* CommandSettings; + const descriptor = yield* api.value + .findStack({ projectRoot: cliSettings.workdir }) + .pipe(Effect.orElseSucceed(() => Option.none())); + return Option.match(descriptor, { + onNone: () => undefined, + onSome: (value) => value.runtime, + }); + }); + +export class StackRuntimeUnavailableError extends Data.TaggedError("StackRuntimeUnavailableError")<{ + readonly message: string; + readonly suggestion?: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.provideFlags; + } +} + +const RUNTIME_UNAVAILABLE = new StackRuntimeUnavailableError({ + message: "Could not determine the stack runtime.", + suggestion: "Start the stack, or start with --runtime docker.", +}); + +/** Fail instead of treating an unknown engine as Docker. */ +export const stackRequireProjectRuntime: Effect.Effect< + StackRuntime, + StackRuntimeUnavailableError, + CommandSettings +> = Effect.gen(function* () { + const api = yield* Effect.serviceOption(StackApi); + if (Option.isNone(api)) return yield* RUNTIME_UNAVAILABLE; + const cliSettings = yield* CommandSettings; + const descriptor = yield* api.value.findStack({ projectRoot: cliSettings.workdir }).pipe( + Effect.mapError( + (cause) => + new StackRuntimeUnavailableError({ + message: cause.message, + suggestion: RUNTIME_UNAVAILABLE.suggestion, + }), + ), + ); + if (Option.isNone(descriptor)) return yield* RUNTIME_UNAVAILABLE; + return descriptor.value.runtime; +}); + +/** Server major from `status().versions.database` (`17.6.1` → 17). */ +export const parsePostgresServerMajor = (version: string): number | undefined => { + const major = Number.parseInt(version.split(".")[0] ?? "", 10); + return Number.isInteger(major) ? major : undefined; +}; + +/** Running stack Postgres major, or undefined when status is missing. */ +export const stackProjectDatabaseMajor: Effect.Effect = + Effect.gen(function* () { + const api = yield* Effect.serviceOption(StackApi); + if (Option.isNone(api)) return undefined; + const cliSettings = yield* CommandSettings; + const descriptor = yield* api.value + .findStack({ projectRoot: cliSettings.workdir }) + .pipe(Effect.orElseSucceed(() => Option.none())); + if (Option.isNone(descriptor)) return undefined; + const stack = yield* api.value + .openStack(descriptor.value.id) + .pipe(Effect.orElseSucceed(() => undefined)); + if (stack === undefined) return undefined; + const status = yield* stack.status.pipe(Effect.orElseSucceed(() => undefined)); + if (status === undefined || typeof status.versions.database !== "string") return undefined; + return parsePostgresServerMajor(status.versions.database); + }); + +const STACK_NATIVE_ENGINE_MESSAGE = + "The stack backend only supports the pg-delta engine. Do not pass --use-migra, --use-pgadmin, --use-pg-schema, or --diff-engine migra."; + +export class StackNativeEngineError extends Data.TaggedError("StackNativeEngineError")<{ + readonly message: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.provideFlags; + } +} + +export const stackRejectNativeDockerDiffEngine: Effect.Effect = + Effect.gen(function* () { + const backend = yield* currentStackBackend; + if (backend.kind !== "stack") return; + return yield* new StackNativeEngineError({ message: STACK_NATIVE_ENGINE_MESSAGE }); + }); + +export const stackLocalDatabaseUrl: Effect.Effect = + Effect.gen(function* () { + const opened = yield* openProjectStack; + if (Option.isNone(opened)) return yield* notRunning(); + const credentials = yield* opened.value.stack.credentials.pipe( + Effect.mapError((cause) => notRunning(cause.message)), + ); + return Redacted.value(credentials.database.url); + }); + +export const stackLocalDatabaseConn: Effect.Effect< + PgConnInput, + LocalDbRunningError, + CommandSettings +> = Effect.gen(function* () { + const url = yield* stackLocalDatabaseUrl; + const conn = parseConnectionString(url); + if (conn === undefined) { + return yield* notRunning(`failed to parse stack database URL`); + } + return conn; +}); + +/** + * Start a postgres-only stack for `db start` and declarative local ensure. Fresh stacks persist + * the overlay; an existing full project stack is started without rewriting `--exclude`. + */ +export const stackEnsurePostgresOnlyStarted: Effect.Effect< + "already-running" | "started", + LocalDbRunningError, + CommandSettings | FileSystem.FileSystem | Path.Path | Output +> = Effect.gen(function* () { + const api = yield* Effect.serviceOption(StackApi); + if (Option.isNone(api)) return yield* startFailed({ message: "stack API is unavailable" }); + const cliSettings = yield* CommandSettings; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* loadStackConfig(cliSettings.workdir).pipe(Effect.mapError(startFailed)); + const applyCatalog = (stack: EffectStack) => + Effect.gen(function* () { + const catalog = yield* Effect.serviceOption(StackCatalogSetup); + if (Option.isNone(catalog)) + return yield* startFailed({ message: "stack catalog setup is unavailable" }); + const toml = yield* readDbToml(fs, path, cliSettings.workdir).pipe( + Effect.mapError(startFailed), + ); + yield* catalog.value + .apply({ + target: { + kind: "live", + stack, + projectRoot: cliSettings.workdir, + config, + }, + overlay: { + webhooks: "config", + webhooksEnabled: toml.webhooksEnabled, + apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, + vault: toml.vault, + workdir: cliSettings.workdir, + }, + }) + .pipe(Effect.mapError(startFailed)); + }); + const existing = yield* api.value + .findStack({ projectRoot: cliSettings.workdir }) + .pipe(Effect.mapError(startFailed)); + if (Option.isNone(existing) || existing.value.desiredLifecycle === "unconfigured") { + const stack = Option.isNone(existing) + ? yield* api.value + .createStack({ projectRoot: cliSettings.workdir }) + .pipe(Effect.mapError(startFailed)) + : yield* api.value.openStack(existing.value.id).pipe(Effect.mapError(startFailed)); + yield* stack + .start({ config: postgresOnlyStackStartConfig(config) }) + .pipe(Effect.mapError(startFailed)); + yield* applyCatalog(stack); + return "started"; + } + const stack = yield* api.value.openStack(existing.value.id).pipe(Effect.mapError(startFailed)); + const status = yield* stack.status.pipe(Effect.mapError(startFailed)); + const database = status.capabilities.find((capability) => capability.name === "database"); + if (status.lifecycle === "running" && database?.state === "ready") { + yield* applyCatalog(stack); + return "already-running"; + } + yield* stack.start().pipe(Effect.mapError(startFailed)); + yield* applyCatalog(stack); + return "started"; +}); diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts new file mode 100644 index 0000000000..97dd3ed87b --- /dev/null +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -0,0 +1,429 @@ +import { CliConfigSchema, type CliConfig } from "@supabase/config"; +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Deferred, Effect, Fiber, FileSystem, Layer, Option, Path, Redacted, Schema } from "effect"; +import { + EphemeralPostgresError, + databaseBootstrapIdentity, + type CreateEphemeralPostgresOptions, + type EffectEphemeralPostgres, +} from "@supabase/stack/effect"; +import { mockOutput } from "../../tests/helpers/mocks.ts"; +import { + mockCommandSettings, + useTempWorkdir, + withEnvVar, +} from "../../tests/helpers/command-mocks.ts"; +import { SHADOW_CACHE_ENV } from "./db-bootstrap/shadow-cache.ts"; +import { DbConnection } from "./db-connection.service.ts"; +import { stackBackendLayer } from "./stack-backend.ts"; +import { + StackEphemeralPostgres, + stackAcquireShadowDatabase, + stackShadowBaselineTarFileName, + stackShadowCacheKey, +} from "./stack-shadow.ts"; +import type { ShadowSetupInput } from "./db-bootstrap/shadow-database.ts"; +import { + noopStackCatalogSetupLayer, + recordingStackCatalogSetup, + StackCatalogSetup, +} from "./stack-catalog-setup.ts"; + +const tmp = useTempWorkdir("stack-shadow-"); +const defaultConfig: CliConfig = Schema.decodeSync(CliConfigSchema)({}); + +const mockEphemeral = () => { + const restores: Array = []; + const exports: Array = []; + const create = ( + options: CreateEphemeralPostgresOptions, + ): Effect.Effect => + Effect.sync(() => { + restores.push(options.restoreFrom); + return { + host: "127.0.0.1", + port: 59999, + version: "17.6.1", + runtime: { kind: "native" as const }, + artifactIdentity: "native:17.6.1", + url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), + start: Effect.void, + stop: Effect.void, + exportPgData: (tarPath: string) => + Effect.gen(function* () { + exports.push(tarPath); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(tarPath, "pgdata").pipe(Effect.ignore); + }), + }; + }); + return { + restores, + exports, + layer: Layer.succeed(StackEphemeralPostgres, { + create, + resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), + }), + }; +}; + +const db = Layer.succeed(DbConnection, { + connect: () => + Effect.succeed({ + exec: () => Effect.void, + query: () => Effect.succeed([]), + execBatch: () => Effect.void, + extensionExists: () => Effect.succeed(false), + copyToCsv: () => Effect.succeed(new Uint8Array()), + queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), + }), +}); + +const input = (fs: FileSystem.FileSystem, path: Path.Path): ShadowSetupInput => ({ + db: { major_version: 17, settings: {} }, + experimental: defaultConfig.experimental, + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwtExpiry: 3600, + networkId: "n", + image: "stack-ephemeral", + configImage: "stack-ephemeral", + shadowPort: 54320, + password: "postgres", + projectId: "proj", + isBitbucketPipeline: false, + workdir: tmp.current, + extraHosts: [], + fs, + path, + hostname: "127.0.0.1", + healthTimeoutSeconds: 2, + setup: { + majorVersion: 17, + config: defaultConfig, + dbUrl: "postgresql://postgres:postgres@127.0.0.1:54320/postgres", + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwks: Effect.succeed("{}"), + apiUrl: "http://127.0.0.1:54321", + authExternalUrl: undefined, + siteUrl: "http://127.0.0.1:3000", + anonKey: "anon", + serviceRoleKey: "service", + storageTargetMigration: "", + realtimeEnabledForSetup: false, + storageEnabledForSetup: false, + authEnabledForSetup: false, + serviceVersionOverrides: {}, + projectEnvValues: undefined, + debug: false, + webhooksEnabled: false, + apiAutoExposeNewTables: Option.none(), + vault: [], + }, +}); + +const withShadowCacheHome = ( + home: string, + value: string, + body: Effect.Effect, +): Effect.Effect => + withEnvVar("SUPABASE_HOME", home, withEnvVar(SHADOW_CACHE_ENV, value, body)); + +const expectedCacheKey = () => + stackShadowCacheKey({ + artifactIdentity: "native:17.6.1", + majorVersion: 17, + runtimeKind: "native", + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwtExpiry: 3600, + dbPassword: "postgres", + dbSettings: {}, + rolesSql: "", + bootstrapIdentity: databaseBootstrapIdentity, + webhooksEnabled: false, + apiGrantsKept: true, + vault: [], + jwks: "", + storageTargetMigration: "", + authEnabled: false, + storageEnabled: false, + realtimeEnabled: false, + authArtifact: "", + storageArtifact: "", + realtimeArtifact: "", + }); + +describe("stackAcquireShadowDatabase", () => { + it.live( + "exports a stack-shadow-baseline tar on a cold miss and restores it on a warm hit", + () => { + const ephemeral = mockEphemeral(); + const out = mockOutput(); + const catalog = recordingStackCatalogSetup((input) => input.target.kind); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped(); + return yield* withShadowCacheHome( + home, + "1", + Effect.gen(function* () { + const first = yield* stackAcquireShadowDatabase(input(fs, path)); + expect(first.baselinePresent).toBe(false); + expect(catalog.applied).toEqual(["ephemeral"]); + expect(first.artifactIdentity).toBe("native:17.6.1"); + expect(ephemeral.restores).toEqual([undefined]); + expect(ephemeral.exports).toHaveLength(1); + expect(ephemeral.exports[0]?.endsWith(`.${String(process.pid)}.partial`)).toBe(true); + const names = (yield* fs.readDirectory( + path.join(home, "cache", "shadow-baseline"), + )).filter((name) => name.endsWith(".tar") && !name.includes(".partial")); + expect(names).toHaveLength(1); + const info = yield* fs.stat(path.join(home, "cache", "shadow-baseline", names[0]!)); + expect((Number(info.mode) & 0o777).toString(8)).toBe("600"); + expect(names[0]?.startsWith("stack-shadow-baseline-")).toBe(true); + expect(names[0]).toBe(stackShadowBaselineTarFileName(expectedCacheKey())); + + const warm = yield* stackAcquireShadowDatabase(input(fs, path)); + expect(warm.baselinePresent).toBe(true); + expect(ephemeral.restores[1]?.endsWith(names[0] ?? "")).toBe(true); + expect(catalog.applied).toEqual(["ephemeral"]); + }), + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + out.layer, + db, + mockCommandSettings({ workdir: tmp.current }), + stackBackendLayer("stack"), + ephemeral.layer, + catalog.layer, + ), + ), + ); + }, + ); + + it.live("skips the cache when SUPABASE_SHADOW_CACHE is 0", () => { + const ephemeral = mockEphemeral(); + const out = mockOutput(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped(); + return yield* withShadowCacheHome( + home, + "0", + Effect.gen(function* () { + const handle = yield* stackAcquireShadowDatabase(input(fs, path)); + expect(handle.baselinePresent).toBe(false); + expect(ephemeral.exports).toHaveLength(0); + const names = yield* fs + .readDirectory(path.join(home, "cache", "shadow-baseline")) + .pipe(Effect.orElseSucceed(() => [])); + expect(names.filter((name) => name.endsWith(".tar"))).toEqual([]); + }), + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + out.layer, + db, + mockCommandSettings({ workdir: tmp.current }), + stackBackendLayer("stack"), + ephemeral.layer, + noopStackCatalogSetupLayer, + ), + ), + ); + }); + + it.live("keeps the cluster uncached when the baseline export fails", () => { + const restores: Array = []; + const out = mockOutput(); + const layer = Layer.succeed(StackEphemeralPostgres, { + create: (options) => + Effect.sync(() => { + restores.push(options.restoreFrom); + return { + host: "127.0.0.1", + port: 59999, + version: "17.6.1", + runtime: { kind: "native" as const }, + artifactIdentity: "native:17.6.1", + url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), + start: Effect.void, + stop: Effect.void, + exportPgData: () => + Effect.fail( + new EphemeralPostgresError({ message: "export failed", reason: "snapshot" }), + ), + }; + }), + resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), + }); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped(); + return yield* withShadowCacheHome( + home, + "1", + Effect.gen(function* () { + const handle = yield* stackAcquireShadowDatabase(input(fs, path)); + expect(handle.baselinePresent).toBe(false); + expect(handle.snapshotKey).toBeUndefined(); + expect(restores).toEqual([undefined]); + expect(out.stderrText).toContain("Warning: shadow baseline not cached:"); + const names = yield* fs + .readDirectory(path.join(home, "cache", "shadow-baseline")) + .pipe(Effect.orElseSucceed(() => [])); + expect( + names.filter((name) => name.endsWith(".tar") && !name.includes(".partial")), + ).toEqual([]); + }), + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + out.layer, + db, + mockCommandSettings({ workdir: tmp.current }), + stackBackendLayer("stack"), + layer, + noopStackCatalogSetupLayer, + ), + ), + ); + }); + + it.live("warns and cold-provisions when a cached baseline restore fails", () => { + const restores: Array = []; + const out = mockOutput(); + const layer = Layer.succeed(StackEphemeralPostgres, { + create: (options) => { + restores.push(options.restoreFrom); + if (options.restoreFrom !== undefined) + return Effect.fail( + new EphemeralPostgresError({ + message: "restore failed", + reason: "restore-mismatch", + }), + ); + return Effect.succeed({ + host: "127.0.0.1", + port: 59999, + version: "17.6.1", + runtime: { kind: "native" as const }, + artifactIdentity: "native:17.6.1", + url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), + start: Effect.void, + stop: Effect.void, + exportPgData: () => Effect.void, + }); + }, + resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), + }); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped(); + const cacheDir = path.join(home, "cache", "shadow-baseline"); + yield* fs.makeDirectory(cacheDir, { recursive: true }); + const tarName = stackShadowBaselineTarFileName(expectedCacheKey()); + yield* fs.writeFileString(path.join(cacheDir, tarName), "corrupt"); + return yield* withShadowCacheHome( + home, + "1", + Effect.gen(function* () { + const handle = yield* stackAcquireShadowDatabase(input(fs, path)); + expect(handle.baselinePresent).toBe(false); + expect(restores).toHaveLength(2); + expect(restores[0]?.endsWith(tarName)).toBe(true); + expect(restores[1]).toBeUndefined(); + expect(out.stderrText).toContain("Warning: shadow baseline not cached: restore failed"); + }), + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + out.layer, + db, + mockCommandSettings({ workdir: tmp.current }), + stackBackendLayer("stack"), + layer, + noopStackCatalogSetupLayer, + ), + ), + ); + }); + + it.live("stops the cluster when acquire is interrupted during catalog overlay", () => { + const out = mockOutput(); + return Effect.scoped( + Effect.gen(function* () { + const started = yield* Deferred.make(); + let stopped = false; + const ephemeral = Layer.succeed(StackEphemeralPostgres, { + create: () => + Effect.sync(() => ({ + host: "127.0.0.1", + port: 59999, + version: "17.6.1", + runtime: { kind: "native" as const }, + artifactIdentity: "native:17.6.1", + url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), + start: Effect.void, + stop: Effect.sync(() => { + stopped = true; + }), + exportPgData: () => Effect.die("export should not run"), + })), + resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), + }); + const catalog = Layer.succeed(StackCatalogSetup, { + apply: () => + Effect.gen(function* () { + yield* Deferred.succeed(started, undefined); + return yield* Effect.never; + }), + }); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped(); + const fiber = yield* Effect.forkChild( + withShadowCacheHome(home, "0", stackAcquireShadowDatabase(input(fs, path))).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + BunServices.layer, + out.layer, + db, + mockCommandSettings({ workdir: tmp.current }), + stackBackendLayer("stack"), + ephemeral, + catalog, + ), + ), + ), + ); + yield* Deferred.await(started); + yield* Fiber.interrupt(fiber); + expect(stopped).toBe(true); + }), + ).pipe(Effect.provide(BunServices.layer)); + }); +}); diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts new file mode 100644 index 0000000000..d6c946be74 --- /dev/null +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -0,0 +1,696 @@ +import { scryptSync } from "node:crypto"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- pid scopes the exclusive temp name across processes. +import process from "node:process"; +import { + Clock, + Context, + Crypto, + Effect, + FileSystem, + Layer, + Option, + Path, + Predicate, + Redacted, + Result, + Scope, + Semaphore, +} from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { + createEphemeralPostgres, + databaseBootstrapIdentity, + resolveEphemeralPostgresRelease, + schemaInitArtifactIdentity, + type CreateEphemeralPostgresOptions, + type EffectEphemeralPostgres, + type EphemeralPostgresRelease, + type EphemeralPostgresSettings, + type SchemaInitCapabilityName, + type StackConfig, + type StackRuntime, + type StackRuntimePreference, + type StackVersionUnsupportedError, +} from "@supabase/stack/effect"; +import { Output } from "../shared/output/output.service.ts"; +import { CommandSettings } from "../config/command-settings.service.ts"; +import { DbConnection } from "./db-connection.service.ts"; +import { shadowBaselineCacheDir } from "./pgdelta.paths.ts"; +import { + SHADOW_BASELINE_KEEP, + SHADOW_BASELINE_MAX_AGE_MS, + SHADOW_CACHE_ENV, + canonicalJson, + shadowBaselineEmbeddedDigest, + shadowBaselineTarsToEvict, + touchShadowBaselineTar, +} from "./db-bootstrap/shadow-cache.ts"; +import { viperEnvBoolWithProjectFallback } from "./viper-env.ts"; +import { + connectShadowDatabase, + ShadowDbError, + type ShadowSetupInput, + type ShadowSourceResult, +} from "./db-bootstrap/shadow-database.ts"; +import { listLocalMigrationPaths } from "./migration-history.ts"; +import { applyMigrations } from "./migration-apply.ts"; +import { stackProjectRuntime } from "./stack-local-database.ts"; +import { loadStackConfig } from "./stack-config.ts"; +import { StackCatalogSetup } from "./stack-catalog-setup.ts"; +import { resolveSetupWebhooksEnabled, type SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; +import type { VaultSecret } from "./vault.ts"; + +/** Optional factory so CLI tests can `Layer.succeed` a fake cluster. */ +export class StackEphemeralPostgres extends Context.Service< + StackEphemeralPostgres, + { + readonly create: typeof createEphemeralPostgres; + readonly resolveRelease: typeof resolveEphemeralPostgresRelease; + } +>()("supabase/experimental-stack/EphemeralPostgres") {} + +export const ephemeralPostgresLayer = Layer.succeed(StackEphemeralPostgres, { + create: createEphemeralPostgres, + resolveRelease: resolveEphemeralPostgresRelease, +}); + +const TAR_PREFIX = "stack-shadow-baseline-"; + +/** A partial older than 5 minutes is abandoned; a live export finishes in seconds. */ +const STACK_SHADOW_PARTIAL_ABANDON_MS = 5 * 60 * 1000; + +export const stackShadowBaselineTarFileName = (key: string): string => `${TAR_PREFIX}${key}.tar`; + +const isStackShadowBaselineTar = (fileName: string): boolean => + /^stack-shadow-baseline-[0-9a-f]{16}\.tar$/u.test(fileName); + +export function isStackShadowBaselinePartial(fileName: string): boolean { + return /^stack-shadow-baseline-[0-9a-f]{16}\.tar\.\d+\.partial$/u.test(fileName); +} + +const stackShadowExportMutex = Semaphore.makeUnsafe(1); + +export interface StackShadowCacheKeyInputs { + readonly artifactIdentity: string; + readonly majorVersion: number; + readonly runtimeKind: string; + readonly jwtSecret: string; + readonly jwtExpiry: number; + readonly dbPassword: string; + readonly dbSettings: unknown; + readonly rolesSql: string; + readonly bootstrapIdentity: string; + readonly webhooksEnabled: boolean; + readonly apiGrantsKept: boolean; + readonly vault: ReadonlyArray; + readonly jwks: string; + readonly storageTargetMigration: string; + readonly authEnabled: boolean; + readonly storageEnabled: boolean; + readonly realtimeEnabled: boolean; + readonly authArtifact: string; + readonly storageArtifact: string; + readonly realtimeArtifact: string; +} + +export const stackShadowCacheKey = (inputs: StackShadowCacheKeyInputs): string => { + const quoted = (value: string) => JSON.stringify(value); + const lines: Array = [ + `artifact=${quoted(inputs.artifactIdentity)}`, + `major_version=${inputs.majorVersion}`, + `runtime=${quoted(inputs.runtimeKind)}`, + `jwt_secret=${quoted(inputs.jwtSecret)}`, + `jwt_expiry=${inputs.jwtExpiry}`, + `db_password=${quoted(inputs.dbPassword)}`, + `db_settings=${canonicalJson(inputs.dbSettings ?? {})}`, + `bootstrap=${quoted(inputs.bootstrapIdentity)}`, + `api_grants_kept=${inputs.apiGrantsKept}`, + `webhooks_enabled=${inputs.webhooksEnabled}`, + `baseline_embedded_digest=${shadowBaselineEmbeddedDigest()}`, + `schema_init=auth=${inputs.authEnabled},storage=${inputs.storageEnabled},realtime=${inputs.realtimeEnabled}`, + inputs.authEnabled ? `auth_artifact=${quoted(inputs.authArtifact)}` : "auth_artifact=excluded", + inputs.storageEnabled + ? `storage_artifact=${quoted(inputs.storageArtifact)}` + : "storage_artifact=excluded", + inputs.realtimeEnabled + ? `realtime_artifact=${quoted(inputs.realtimeArtifact)}` + : "realtime_artifact=excluded", + inputs.realtimeEnabled && inputs.majorVersion >= 15 + ? `realtime_jwks=${quoted(inputs.jwks)}` + : "realtime_jwks=excluded", + inputs.storageEnabled && inputs.majorVersion >= 15 + ? `storage_target_migration=${quoted(inputs.storageTargetMigration)}` + : "storage_target_migration=excluded", + ]; + for (const secret of inputs.vault + .filter((secret) => secret.resolved) + .sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0))) { + lines.push(`vault=${JSON.stringify([secret.name, secret.value])}`); + } + return scryptSync( + `${lines.join("\n")}\nroles_sql=\n${inputs.rolesSql}`, + "supabase-stack-shadow-cache-key", + 32, + ) + .toString("hex") + .slice(0, 16); +}; + +const capabilityPinVersion = ( + cap: { readonly enabled?: boolean; readonly version?: string } | undefined, +): string | undefined => (cap === undefined || cap.enabled === false ? undefined : cap.version); + +const trioSchemaInitArtifact = ( + enabled: boolean, + name: Extract, + config: StackConfig | undefined, +): string => { + if (!enabled) return ""; + return ( + schemaInitArtifactIdentity(name, capabilityPinVersion(config?.capabilities?.[name])) ?? + "missing" + ); +}; + +export interface StackShadowAcquiredHandle { + readonly url: string; + readonly host: string; + readonly port: number; + readonly artifactIdentity: string; + readonly runtime: StackRuntime; + readonly baselinePresent: boolean; + readonly snapshotKey?: string; + readonly ephemeral: EffectEphemeralPostgres; +} + +export interface StackShadowAcquireOpts { + readonly bypassCache?: boolean; + readonly port?: number; + readonly runtime?: StackRuntimePreference; + readonly webhooks?: SetupDatabaseOptions["webhooks"]; +} + +const cacheEnabled = (projectEnv: Record | undefined, bypass: boolean): boolean => + !bypass && + viperEnvBoolWithProjectFallback(SHADOW_CACHE_ENV, projectEnv ?? {}, { + whenUnset: true, + }); + +const readRolesSql = ( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, +): Effect.Effect => + fs.readFileString(path.join(workdir, "supabase", "roles.sql")).pipe( + Effect.catchTag("PlatformError", (error) => + Predicate.isTagged(error.reason, "NotFound") + ? Effect.succeed("") + : Effect.fail( + new ShadowDbError({ + message: `failed to read supabase/roles.sql: ${error.message}`, + reason: "filesystem", + }), + ), + ), + ); + +const runtimePreference = ( + runtime: StackRuntime | undefined, + override?: StackRuntimePreference, +): StackRuntimePreference | undefined => { + if (override !== undefined) return override; + if (runtime === undefined) return undefined; + return runtime.kind === "native" + ? { kind: "native" } + : { kind: "container", engine: runtime.engine }; +}; + +const postgresSettings = (value: unknown): EphemeralPostgresSettings | undefined => { + if (value === undefined || value === null || typeof value !== "object" || Array.isArray(value)) + return undefined; + return Object.fromEntries( + Object.entries(value).filter( + (entry): entry is [string, string | number | boolean] => + typeof entry[1] === "string" || + typeof entry[1] === "number" || + typeof entry[1] === "boolean", + ), + ); +}; + +const createOptions = ( + input: ShadowSetupInput, + runtime: StackRuntimePreference | undefined, + restoreFrom: string | undefined, + port: number | undefined, +): CreateEphemeralPostgresOptions => ({ + databasePassword: Redacted.make(input.password), + jwtSecret: Redacted.make(input.jwtSecret), + jwtExpiry: input.jwtExpiry, + postgresSettings: postgresSettings(input.db.settings), + healthTimeout: `${String(input.healthTimeoutSeconds)}s`, + version: String(input.setup.majorVersion), + ...(runtime === undefined ? {} : { runtime }), + ...(port === undefined ? {} : { port }), + ...(restoreFrom === undefined ? {} : { restoreFrom }), +}); + +const connFrom = (handle: EffectEphemeralPostgres, password: string) => ({ + host: handle.host, + port: handle.port, + user: "postgres", + password, + database: "postgres", +}); + +const applyColdCatalog = ( + handle: EffectEphemeralPostgres, + input: ShadowSetupInput, + webhooks: SetupDatabaseOptions["webhooks"], +): Effect.Effect => + Effect.gen(function* () { + const catalog = yield* Effect.serviceOption(StackCatalogSetup); + if (Option.isNone(catalog)) + return yield* new ShadowDbError({ + message: "stack catalog setup is unavailable", + reason: "database", + }); + const config = yield* loadStackConfig(input.workdir).pipe( + Effect.mapError( + (cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" }), + ), + ); + yield* catalog.value + .apply({ + target: { + kind: "ephemeral", + projectRoot: input.workdir, + runtime: handle.runtime, + config, + databaseUrl: Redacted.value(handle.url), + databasePassword: Redacted.make(input.password), + jwtSecret: Redacted.make(input.jwtSecret), + ...(handle.networkId === undefined ? {} : { networkId: handle.networkId }), + }, + overlay: { + webhooks, + webhooksEnabled: input.setup.webhooksEnabled, + apiAutoExposeNewTables: input.setup.apiAutoExposeNewTables, + vault: input.setup.vault, + workdir: input.workdir, + announceRoles: false, + }, + }) + .pipe( + Effect.mapError( + (cause) => new ShadowDbError({ message: cause.message, reason: "database" }), + ), + ); + }); + +const artifactIdentityFor = ( + runtime: StackRuntimePreference | undefined, + version: string, + image: string, +): string => + runtime?.kind === "container" + ? `container:${runtime.engine ?? "docker"}:${image}` + : `native:${version}`; + +const sweepAbandonedPartials = ( + fs: FileSystem.FileSystem, + path: Path.Path, + cacheDir: string, +): Effect.Effect => + Effect.gen(function* () { + const names = yield* fs.readDirectory(cacheDir).pipe(Effect.orElseSucceed(() => [])); + const now = yield* Clock.currentTimeMillis; + yield* Effect.forEach( + names.filter(isStackShadowBaselinePartial), + (fileName) => + Effect.gen(function* () { + const filePath = path.join(cacheDir, fileName); + const info = yield* fs.stat(filePath); + const mtime = Option.getOrUndefined(info.mtime); + if (mtime !== undefined && now - mtime.getTime() > STACK_SHADOW_PARTIAL_ABANDON_MS) { + yield* fs.remove(filePath).pipe(Effect.ignore); + } + }).pipe(Effect.ignore), + { discard: true }, + ); + }); + +const sweepCache = ( + fs: FileSystem.FileSystem, + path: Path.Path, + cacheDir: string, + keepName: string | undefined, +): Effect.Effect => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const names = yield* fs.readDirectory(cacheDir).pipe(Effect.orElseSucceed(() => [])); + const entries: Array<{ readonly fileName: string; readonly mtimeMs: number }> = []; + for (const fileName of names) { + if (!isStackShadowBaselineTar(fileName)) continue; + const info = yield* fs.stat(path.join(cacheDir, fileName)).pipe(Effect.option); + if (Option.isNone(info) || Option.isNone(info.value.mtime)) continue; + entries.push({ fileName, mtimeMs: info.value.mtime.value.getTime() }); + } + yield* Effect.forEach( + shadowBaselineTarsToEvict(entries, now, { + keep: SHADOW_BASELINE_KEEP, + maxAgeMs: SHADOW_BASELINE_MAX_AGE_MS, + retainFileName: keepName, + isPublishedTar: isStackShadowBaselineTar, + }), + (fileName) => fs.remove(path.join(cacheDir, fileName)).pipe(Effect.ignore), + { discard: true }, + ); + }); + +const writeStackShadowBaselineTar = ( + fs: FileSystem.FileSystem, + path: Path.Path, + cacheDir: string, + tarPath: string, + exportPgData: (tempPath: string) => Effect.Effect, + skipIfPublished: boolean, +): Effect.Effect => + stackShadowExportMutex.withPermit( + Effect.gen(function* () { + if (skipIfPublished) { + const published = yield* fs.exists(tarPath).pipe(Effect.orElseSucceed(() => false)); + if (published) return; + } + yield* fs.makeDirectory(cacheDir, { recursive: true, mode: 0o700 }).pipe( + Effect.mapError( + (cause) => + new ShadowDbError({ + message: `failed to create ${cacheDir}: ${cause.message}`, + reason: "filesystem", + }), + ), + ); + yield* sweepAbandonedPartials(fs, path, cacheDir); + const tempPath = `${tarPath}.${String(process.pid)}.partial`; + yield* fs.remove(tempPath).pipe(Effect.ignore); + yield* Effect.gen(function* () { + yield* Effect.scoped( + fs.open(tempPath, { flag: "wx", mode: 0o600 }).pipe( + Effect.mapError( + (cause) => + new ShadowDbError({ + message: `failed to create ${tempPath}: ${cause.message}`, + reason: "filesystem", + }), + ), + Effect.asVoid, + ), + ); + yield* exportPgData(tempPath); + yield* fs.chmod(tempPath, 0o600).pipe( + Effect.mapError( + (cause) => + new ShadowDbError({ + message: `failed to restrict ${tempPath}: ${cause.message}`, + reason: "filesystem", + }), + ), + ); + yield* fs.rename(tempPath, tarPath).pipe( + Effect.mapError( + (cause) => + new ShadowDbError({ + message: `failed to publish ${tarPath}: ${cause.message}`, + reason: "filesystem", + }), + ), + ); + }).pipe(Effect.onError(() => fs.remove(tempPath).pipe(Effect.ignore))); + yield* sweepCache(fs, path, cacheDir, path.basename(tarPath)); + }), + ); + +const mapCreateError = (cause: unknown): ShadowDbError => + new ShadowDbError({ + message: + typeof cause === "object" && cause !== null && "message" in cause + ? String(Reflect.get(cause, "message")) + : String(cause), + reason: "database", + }); + +const runtimeKindFor = (runtime: StackRuntimePreference | undefined): string => + runtime?.kind === "container" ? `container:${runtime.engine ?? "docker"}` : "native"; + +const ephemeralApis = (): Effect.Effect<{ + readonly create: typeof createEphemeralPostgres; + readonly resolveRelease: ( + version?: string, + ) => Effect.Effect; +}> => + Effect.serviceOption(StackEphemeralPostgres).pipe( + Effect.map((value) => + Option.getOrElse(value, () => ({ + create: createEphemeralPostgres, + resolveRelease: resolveEphemeralPostgresRelease, + })), + ), + ); + +const ownCluster = (ephemeral: EffectEphemeralPostgres) => + Effect.addFinalizer(() => ephemeral.stop.pipe(Effect.ignore)); + +export const stackAcquireShadowDatabase = ( + input: ShadowSetupInput, + opts: StackShadowAcquireOpts = {}, +): Effect.Effect< + StackShadowAcquiredHandle, + ShadowDbError | E, + | Output + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner + | Scope.Scope + | CommandSettings +> => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const apis = yield* ephemeralApis(); + const projectRuntime = yield* stackProjectRuntime; + const runtime = runtimePreference(projectRuntime, opts.runtime); + const rolesSql = yield* readRolesSql(input.fs, input.path, input.workdir); + const cacheOn = cacheEnabled(input.setup.projectEnvValues, opts.bypassCache === true); + const cacheDir = shadowBaselineCacheDir(path); + const webhooks = opts.webhooks; + yield* fs.makeDirectory(cacheDir, { recursive: true, mode: 0o700 }).pipe(Effect.ignore); + + const startEmpty = () => + apis + .create(createOptions(input, runtime, undefined, opts.port)) + .pipe(Effect.mapError(mapCreateError)); + + if (!cacheOn) { + const ephemeral = yield* startEmpty(); + yield* ownCluster(ephemeral); + yield* applyColdCatalog(ephemeral, input, webhooks); + return { + url: Redacted.value(ephemeral.url), + host: ephemeral.host, + port: ephemeral.port, + artifactIdentity: ephemeral.artifactIdentity, + runtime: ephemeral.runtime, + baselinePresent: false, + ephemeral, + }; + } + + const jwks = + input.setup.realtimeEnabledForSetup && input.setup.majorVersion >= 15 + ? yield* input.setup.jwks + : ""; + const release = yield* apis + .resolveRelease(String(input.setup.majorVersion)) + .pipe(Effect.mapError(mapCreateError)); + const identity = artifactIdentityFor(runtime, release.version, release.image); + const trioEnabled = + input.setup.authEnabledForSetup || + input.setup.storageEnabledForSetup || + input.setup.realtimeEnabledForSetup; + const stackConfig = trioEnabled + ? yield* loadStackConfig(input.workdir).pipe( + Effect.mapError( + (cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" }), + ), + ) + : undefined; + const key = stackShadowCacheKey({ + artifactIdentity: identity, + majorVersion: input.setup.majorVersion, + runtimeKind: runtimeKindFor(runtime), + jwtSecret: input.jwtSecret, + jwtExpiry: input.jwtExpiry, + dbPassword: input.password, + dbSettings: input.db.settings, + rolesSql, + bootstrapIdentity: databaseBootstrapIdentity, + webhooksEnabled: resolveSetupWebhooksEnabled(webhooks, input.setup.webhooksEnabled), + apiGrantsKept: Option.getOrElse(input.setup.apiAutoExposeNewTables, () => true), + vault: input.setup.vault, + jwks, + storageTargetMigration: input.setup.storageTargetMigration, + authEnabled: input.setup.authEnabledForSetup, + storageEnabled: input.setup.storageEnabledForSetup, + realtimeEnabled: input.setup.realtimeEnabledForSetup, + authArtifact: trioSchemaInitArtifact(input.setup.authEnabledForSetup, "auth", stackConfig), + storageArtifact: trioSchemaInitArtifact( + input.setup.storageEnabledForSetup, + "storage", + stackConfig, + ), + realtimeArtifact: trioSchemaInitArtifact( + input.setup.realtimeEnabledForSetup, + "realtime", + stackConfig, + ), + }); + const tarName = stackShadowBaselineTarFileName(key); + const tarPath = path.join(cacheDir, tarName); + const cached = yield* fs.exists(tarPath).pipe(Effect.orElseSucceed(() => false)); + yield* sweepAbandonedPartials(fs, path, cacheDir); + yield* sweepCache(fs, path, cacheDir, tarName); + + if (cached) { + const restored = yield* Effect.result( + apis.create(createOptions(input, runtime, tarPath, opts.port)), + ); + if (Result.isSuccess(restored)) { + yield* ownCluster(restored.success); + yield* touchShadowBaselineTar(fs, tarPath); + return { + url: Redacted.value(restored.success.url), + host: restored.success.host, + port: restored.success.port, + artifactIdentity: restored.success.artifactIdentity, + runtime: restored.success.runtime, + baselinePresent: true, + snapshotKey: key, + ephemeral: restored.success, + }; + } + const output = yield* Output; + yield* output.raw( + `Warning: shadow baseline not cached: ${restored.failure.message}\n`, + "stderr", + ); + } + + const probe = yield* startEmpty(); + yield* ownCluster(probe); + yield* applyColdCatalog(probe, input, webhooks); + const exported = yield* Effect.result( + Effect.gen(function* () { + const rolesSqlNow = yield* readRolesSql(input.fs, input.path, input.workdir); + if (rolesSqlNow !== rolesSql) { + return yield* new ShadowDbError({ + message: "supabase/roles.sql changed during provisioning", + reason: "filesystem", + }); + } + yield* probe.stop.pipe(Effect.mapError(mapCreateError)); + yield* writeStackShadowBaselineTar( + fs, + path, + cacheDir, + tarPath, + (tempPath) => probe.exportPgData(tempPath).pipe(Effect.mapError(mapCreateError)), + !cached, + ); + }), + ); + yield* probe.start.pipe(Effect.mapError(mapCreateError)); + if (Result.isFailure(exported)) { + const output = yield* Output; + yield* output.raw( + `Warning: shadow baseline not cached: ${exported.failure.message}\n`, + "stderr", + ); + } + return { + url: Redacted.value(probe.url), + host: probe.host, + port: probe.port, + artifactIdentity: probe.artifactIdentity, + runtime: probe.runtime, + baselinePresent: false, + snapshotKey: Result.isSuccess(exported) ? key : undefined, + ephemeral: probe, + }; + }); + +export const stackWithShadowDatabase = ( + input: ShadowSetupInput, + use: (handle: StackShadowAcquiredHandle) => Effect.Effect, + opts: StackShadowAcquireOpts = {}, +): Effect.Effect< + A, + E2 | ShadowDbError | E, + | R2 + | Output + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner + | CommandSettings +> => + Effect.scoped( + Effect.gen(function* () { + const handle = yield* stackAcquireShadowDatabase(input, opts); + return yield* use(handle); + }), + ); + +export const stackPrepareShadowSource = ( + handle: StackShadowAcquiredHandle, + input: ShadowSetupInput, +): Effect.Effect< + Pick, + ShadowDbError, + DbConnection | Output | Scope.Scope | FileSystem.FileSystem | Path.Path +> => + stackMigrateShadow(handle, input).pipe( + Effect.as({ sourceUrl: handle.url, targetUrlOverride: undefined }), + ); + +export const stackMigrateShadow = ( + handle: StackShadowAcquiredHandle, + input: ShadowSetupInput, +): Effect.Effect< + void, + ShadowDbError, + DbConnection | Output | Scope.Scope | FileSystem.FileSystem | Path.Path +> => + Effect.scoped( + Effect.gen(function* () { + const migrationsDir = input.path.join(input.workdir, "supabase", "migrations"); + const pending = yield* listLocalMigrationPaths(input.fs, input.path, migrationsDir).pipe( + Effect.mapError( + (cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" }), + ), + ); + const session = yield* connectShadowDatabase(connFrom(handle.ephemeral, input.password)); + yield* applyMigrations( + session, + input.fs, + input.path, + pending, + (message) => new ShadowDbError({ message, reason: "database" }), + ).pipe( + Effect.catchTag("DbConnectError", (cause) => + Effect.fail(new ShadowDbError({ message: cause.message, reason: "connect" })), + ), + ); + }), + ); diff --git a/apps/cli/src/command-internal/stack-shadow.unit.test.ts b/apps/cli/src/command-internal/stack-shadow.unit.test.ts new file mode 100644 index 0000000000..e6a021bc85 --- /dev/null +++ b/apps/cli/src/command-internal/stack-shadow.unit.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + stackShadowBaselineTarFileName, + stackShadowCacheKey, + isStackShadowBaselinePartial, +} from "./stack-shadow.ts"; + +const overlay = { + webhooksEnabled: false, + apiGrantsKept: true, + vault: [] as const, + jwks: "", + storageTargetMigration: "", + authEnabled: false, + storageEnabled: false, + realtimeEnabled: false, + authArtifact: "", + storageArtifact: "", + realtimeArtifact: "", +}; + +const base = { + artifactIdentity: "native:17.6.1", + majorVersion: 17, + runtimeKind: "native", + jwtSecret: "jwt", + jwtExpiry: 3600, + dbPassword: "postgres", + dbSettings: {}, + rolesSql: "", + bootstrapIdentity: "bootstrap-v1", + ...overlay, +}; + +describe("stackShadowCacheKey", () => { + it("changes when the artifact identity or runtime kind changes", () => { + const native = stackShadowCacheKey(base); + const otherArtifact = stackShadowCacheKey({ + ...base, + artifactIdentity: "native:17.6.2", + }); + const container = stackShadowCacheKey({ + ...base, + artifactIdentity: "container:docker:example", + runtimeKind: "container:docker", + }); + expect(native).toMatch(/^[0-9a-f]{16}$/u); + expect(native).not.toBe(otherArtifact); + expect(native).not.toBe(container); + expect(stackShadowBaselineTarFileName(native)).toBe(`stack-shadow-baseline-${native}.tar`); + expect(stackShadowBaselineTarFileName(native)).not.toContain("shadow-baseline-shadow"); + }); + + it("recognizes only this module's own partial temp files as abandoned-sweep candidates", () => { + const key = "0123456789abcdef"; + expect(isStackShadowBaselinePartial(`stack-shadow-baseline-${key}.tar.4242.partial`)).toBe( + true, + ); + for (const other of [ + stackShadowBaselineTarFileName(key), + `shadow-baseline-${key}.tar.4242.partial`, + `stack-shadow-baseline-${key}.tar.partial`, + `stack-shadow-baseline-${key}.tar.4242.partial.bak`, + "catalog-local-migrations-abc-123.json", + ]) { + expect(isStackShadowBaselinePartial(other), other).toBe(false); + } + }); + + it("changes when roles.sql, db settings, or bootstrap identity change", () => { + const withRoles = stackShadowCacheKey({ ...base, rolesSql: "create role x;" }); + expect(stackShadowCacheKey(base)).not.toBe(withRoles); + expect(stackShadowCacheKey({ ...base, dbSettings: { max_connections: 20 } })).not.toBe( + stackShadowCacheKey(base), + ); + expect(stackShadowCacheKey({ ...base, bootstrapIdentity: "bootstrap-v2" })).not.toBe( + stackShadowCacheKey(base), + ); + }); + + it("changes when overlay or schema-init membership changes", () => { + expect(stackShadowCacheKey({ ...base, webhooksEnabled: true })).not.toBe( + stackShadowCacheKey(base), + ); + expect(stackShadowCacheKey({ ...base, apiGrantsKept: false })).not.toBe( + stackShadowCacheKey(base), + ); + expect( + stackShadowCacheKey({ + ...base, + vault: [{ name: "a", value: "secret", resolved: true }], + }), + ).not.toBe(stackShadowCacheKey(base)); + expect(stackShadowCacheKey({ ...base, authEnabled: true })).not.toBe(stackShadowCacheKey(base)); + expect( + stackShadowCacheKey({ + ...base, + realtimeEnabled: true, + jwks: '{"keys":[]}', + }), + ).not.toBe(stackShadowCacheKey({ ...base, realtimeEnabled: true, jwks: "{}" })); + expect( + stackShadowCacheKey({ + ...base, + storageEnabled: true, + storageTargetMigration: "20240101000000", + }), + ).not.toBe(stackShadowCacheKey({ ...base, storageEnabled: true })); + expect( + stackShadowCacheKey({ + ...base, + authEnabled: true, + authArtifact: "v2.196.0:ghcr.io/supabase/cli/auth:v2.196.0", + }), + ).not.toBe( + stackShadowCacheKey({ + ...base, + authEnabled: true, + authArtifact: "v2.197.0:ghcr.io/supabase/cli/auth:v2.197.0", + }), + ); + }); +}); diff --git a/apps/cli/src/command-internal/test-db.handler.ts b/apps/cli/src/command-internal/test-db.handler.ts index e1c2ac8e22..bd84b585a7 100644 --- a/apps/cli/src/command-internal/test-db.handler.ts +++ b/apps/cli/src/command-internal/test-db.handler.ts @@ -21,6 +21,13 @@ import { TestDbRunError, } from "./test-db.errors.ts"; import { buildPgProveArgs } from "./test-db.pg-prove-args.ts"; +import { currentStackBackend } from "./stack-backend.ts"; +import { stackProjectDatabaseMajor, stackRequireProjectRuntime } from "./stack-local-database.ts"; +import { + rewriteDumpHostForToolContainer, + requireHostPgProve, + streamHostCommand, +} from "./postgres-client.run.ts"; const ENABLE_PGTAP = "create extension if not exists pgtap with schema extensions"; const DISABLE_PGTAP = "drop extension if exists pgtap"; @@ -101,25 +108,37 @@ export const testDb = Effect.fn("test.db")(function* (flags: TestDbFlags) { debug, }); - // For a local database the pg_prove container joins the supabase docker - // network and reaches postgres via the internal `db:5432` alias; otherwise - // it uses host networking. + const backend = yield* currentStackBackend; + const stackRuntime = + backend.kind === "stack" && isLocal ? yield* stackRequireProjectRuntime : undefined; + const useHostProve = stackRuntime?.kind === "native" && runtimeInfo.platform !== "win32"; + const stackContainerProve = backend.kind === "stack" && isLocal && !useHostProve; + + const networkId = Option.getOrUndefined(networkIdFlag); + const dumpUsesHostNetwork = networkId === undefined || networkId.length === 0; const runEnv = { - PGHOST: isLocal ? "db" : conn.host, - PGPORT: isLocal ? "5432" : String(conn.port), + PGHOST: useHostProve + ? "127.0.0.1" + : stackContainerProve + ? rewriteDumpHostForToolContainer(conn.host, { + platform: runtimeInfo.platform, + usesHostNetwork: dumpUsesHostNetwork, + }) + : isLocal + ? "db" + : conn.host, + PGPORT: isLocal && backend.kind !== "stack" ? "5432" : String(conn.port), PGUSER: conn.user, PGPASSWORD: conn.password, PGDATABASE: conn.database, }; // A non-empty `--network-id` overrides everything (even host mode); - // otherwise local uses the generated `supabase_network_` - // network and remote uses host networking. - const networkId = Option.getOrUndefined(networkIdFlag); + // otherwise local Compose uses `supabase_network_` and remote / stack uses host networking. const network = networkId !== undefined && networkId.length > 0 ? { _tag: "named" as const, name: networkId } - : isLocal + : isLocal && backend.kind !== "stack" ? yield* Effect.gen(function* () { const toml = yield* readDbToml(fs, path, cliSettings.workdir); // The project id is sanitized unconditionally before deriving the @@ -179,9 +198,43 @@ export const testDb = Effect.fn("test.db")(function* (flags: TestDbFlags) { // Docker Desktop provide the mapping natively. const extraHosts = runtimeInfo.platform === "linux" ? ["host.docker.internal:host-gateway"] : []; - // Stream (rather than inherit) stdout so the verdict can be read on the - // way past; every chunk is forwarded byte-exact and unframed. stderr is - // teed live, as inheriting it did. + const onStdout = (chunk: Uint8Array) => + Effect.suspend(() => { + // Split on newlines, carrying the incomplete trailing line into the + // next chunk so a verdict straddling a chunk boundary is still seen. + const lines = (pendingLine + decoder.decode(chunk, { stream: true })).split("\n"); + pendingLine = lines.pop() ?? ""; + for (const line of lines) { + if (line.startsWith(VERDICT_PREFIX)) lastVerdict = line; + else if (FILES_SUMMARY.test(line)) lastSummary = line; + } + return output.rawBytes(chunk, "stdout"); + }); + if (useHostProve) { + const toml = yield* readDbToml(fs, path, cliSettings.workdir); + const expectedMajor = + (backend.kind === "stack" ? yield* stackProjectDatabaseMajor : undefined) ?? + toml.majorVersion; + yield* requireHostPgProve(expectedMajor); + const hostPath = args.hostPaths[0]; + const hostWorkingDir = + hostPath === undefined + ? undefined + : nodePath.extname(hostPath) !== "" + ? nodePath.dirname(hostPath) + : hostPath; + const hostArgs = ["--ext", ".pg", "--ext", ".sql", "-r", ...args.hostPaths]; + if (debug) hostArgs.push("--verbose"); + return yield* streamHostCommand({ + command: "pg_prove", + args: hostArgs, + env: runEnv, + cwd: hostWorkingDir, + onStdout, + teeStderr: true, + captureStderr: false, + }); + } return yield* docker.runStream( { image: getRegistryImageUrl(PG_PROVE_IMAGE), @@ -194,20 +247,7 @@ export const testDb = Effect.fn("test.db")(function* (flags: TestDbFlags) { network, }, { - onStdout: (chunk) => - Effect.suspend(() => { - // Split on newlines, carrying the incomplete trailing line into the - // next chunk so a verdict straddling a chunk boundary is still seen. - const lines = (pendingLine + decoder.decode(chunk, { stream: true })).split("\n"); - pendingLine = lines.pop() ?? ""; - for (const line of lines) { - if (line.startsWith(VERDICT_PREFIX)) lastVerdict = line; - else if (FILES_SUMMARY.test(line)) lastSummary = line; - } - return output.rawBytes(chunk, "stdout"); - }), - // Teed straight to the terminal as inheriting it did; nothing here reads - // the buffered copy, and a pgTAP suite's psql notices are unbounded. + onStdout, teeStderr: true, captureStderr: false, }, @@ -223,7 +263,9 @@ export const testDb = Effect.fn("test.db")(function* (flags: TestDbFlags) { // already streamed to stdout. if (exitCode !== 0) { return yield* Effect.fail( - new TestDbRunError({ message: `error running container: exit ${exitCode}` }), + new TestDbRunError({ + message: `error running ${useHostProve ? "pg_prove" : "container"}: exit ${exitCode}`, + }), ); } diff --git a/apps/cli/src/command-internal/test-db.layers.ts b/apps/cli/src/command-internal/test-db.layers.ts index 52d9b90447..dfe8a84031 100644 --- a/apps/cli/src/command-internal/test-db.layers.ts +++ b/apps/cli/src/command-internal/test-db.layers.ts @@ -8,7 +8,7 @@ import { identityStitchLayer } from "./identity-stitch.ts"; import { debugLoggerLayer } from "./debug-logger.layer.ts"; import { telemetryStateLayer } from "../telemetry/telemetry-state.layer.ts"; import { commandRuntimeLayer } from "../shared/runtime/command-runtime.layer.ts"; - +import { stackApiLayer } from "./stack-api.ts"; /** * Runtime layer shared by `supabase test db` and its hidden alias `supabase * db test`, both calling this same factory and `runTestDbCommand`. @@ -45,5 +45,7 @@ export const testDbRuntimeLayer = (commandPath: ReadonlyArray) => // above, so the lazy linked stack shares a single stitch attempt. identityStitchLayer, telemetryStateLayer, + // Exposed so native-engine prove can read `runtime.kind` and pick PATH pg_prove. + stackApiLayer, commandRuntimeLayer(commandPath), ); diff --git a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md index 569e59e07f..de7605e327 100644 --- a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md @@ -26,22 +26,26 @@ it, and JSON `null` disables formatting without disabling safe compaction. | `/supabase/migrations/*.sql` | SQL | shadow provisioning (applied to the shadow source) — `--use-pgadmin` too, via the SAME `migrateShadowDatabase` | | `/supabase/roles.sql` | SQL | shadow provisioning, PG14 and PG15 alike (unlike `db reset`'s PG15-only local path); also hashed into the shadow-baseline cache key on every cache-eligible acquire, warm hits included (where no baseline is applied at all); missing file tolerated | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | warm shadow-cache hit — the matching snapshot is streamed into the fresh shadow; every cache-eligible acquire (warm hit and successful cold export) also enumerates and `stat`s every `shadow-baseline-*.tar` for LRU keep-3 + 2-day mtime TTL and may delete other keys (`SUPABASE_HOME` overrides the `~/.supabase` root) | +| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar` | tar | `[experimental].stack` only — slim-init + stack bootstrap baseline; key includes artifact identity and runtime kind (native vs container). Never mixed with `shadow-baseline-*.tar` | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | abandoned-partial sweep on every cache-eligible acquire (warm hit and cold export) — enumerated and `stat`ed, and removed when older than 5 minutes (a crashed/SIGKILLed earlier export's leftover) | +| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar..partial` | tar | `[experimental].stack` abandoned-partial sweep — same 5-minute TTL, stack prefix only (legacy `shadow-baseline-*.partial` names are not candidates) | | `[db.migrations].schema_paths` globs / `/supabase/database/**` / `/supabase/schemas/**` | SQL | migra engine only, for the local-target declarative-schema fallback; pg-delta always compares the migrations baseline directly to the live target | | `~/.supabase/access-token` | plain text | `--linked` / `--db-url` with no `SUPABASE_ACCESS_TOKEN` | | `/supabase/.temp/project-ref` | plain text | `--linked` ref resolution — skipped when `--project-ref` (or `SUPABASE_PROJECT_ID`) is set | ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/migrations/_.sql` | SQL | non-empty `--file` diff; bundled pg-delta may emit ordered transaction-aware files, while pgAdmin always emits one | -| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output`; flattened review representation, not a portable apply script | -| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot (native diff targets + the explicit `--from/--to migrations` shadow; never `--use-pgadmin`/`--use-pg-schema`); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | -| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | -| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| Path | Format | When | +| --------------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/migrations/_.sql` | SQL | non-empty `--file` diff; bundled pg-delta may emit ordered transaction-aware files, while pgAdmin always emits one | +| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output`; flattened review representation, not a portable apply script | +| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot (native diff targets + the explicit `--from/--to migrations` shadow; never `--use-pgadmin`/`--use-pg-schema`); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | +| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar` | tar | `[experimental].stack` COLD export of an `EphemeralPostgres` cluster; same LRU keep-3 + 2-day TTL, separate glob so keys cannot collide with legacy SQL-template baselines | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | +| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar..partial` | tar | `[experimental].stack` cold export temp file — pid-scoped, `chmod` 0600, `rename`d into the stack tar above; abandoned leftovers older than 5 minutes are swept on later acquires | +| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | +| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | ## Docker @@ -56,6 +60,11 @@ it, and JSON `null` disables formatting without disabling safe compaction. narrower composition — `createShadowDatabase` -> health-wait -> `migrateShadowDatabase` directly (`diff.handler.ts`'s pgadmin branch) — with no declarative-schema-override branch and no `targetUrlOverride`. +- When `[experimental].stack` / `SUPABASE_EXPERIMENTAL_STACK=1` is on, `--local` inspects the + project stack (`findStack` + `status`, database ready) instead of `supabase_db_`, + and shadows are `@supabase/stack` `EphemeralPostgres` clusters (slim-init baseline, cache + prefix `stack-shadow-baseline-`). Every stack runtime rejects `--use-migra` / `--use-pgadmin` / + `--use-pg-schema`. - `supabase/migra` container — the migra OOM bash fallback only. - **Differ container** (`--use-pgadmin`, CLI-1968) — `supabase/pgadmin-schema-diff:cli-0.0.5` (`dockerfileServiceImage("differ")`). One `docker run --rm` when no `--schema` is given; one diff --git a/apps/cli/src/commands/db/diff/diff.handler.ts b/apps/cli/src/commands/db/diff/diff.handler.ts index 4ca78558f0..5175a73e61 100644 --- a/apps/cli/src/commands/db/diff/diff.handler.ts +++ b/apps/cli/src/commands/db/diff/diff.handler.ts @@ -28,6 +28,16 @@ import { toPostgresURL } from "../../../command-internal/postgres-url.ts"; import { schemaToCsvField } from "../../../command-internal/schema-flags.ts"; import { findDropStatements } from "../../../command-internal/sql-split.ts"; import { buildLocalDbContainerInputs } from "../../../command-internal/db-bootstrap/local-container-inputs.ts"; +import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; +import { StackApi } from "../../../command-internal/stack-api.ts"; +import { + stackLocalDatabaseConn, + stackRejectNativeDockerDiffEngine, +} from "../../../command-internal/stack-local-database.ts"; +import { + stackPrepareShadowSource, + stackWithShadowDatabase, +} from "../../../command-internal/stack-shadow.ts"; import { isLocalDbRunning } from "../../../command-internal/db-bootstrap/local-db-running.ts"; import { waitForHealthyServices } from "../../../command-internal/db-bootstrap/health-check.ts"; import { withShadowDatabase } from "../../../command-internal/db-bootstrap/shadow-cache.ts"; @@ -136,6 +146,7 @@ export const dbDiff = Effect.fn("db.diff")(function* (flags: DbDiffFlags) { const path = yield* Path.Path; const dnsResolver = yield* DnsResolverFlag; const debug = yield* DebugFlag; + const stackApi = yield* Effect.serviceOption(StackApi); // Resolved linked ref, captured so the post-run finalizer caches the project // (GET /v1/projects/{ref}). @@ -168,6 +179,13 @@ export const dbDiff = Effect.fn("db.diff")(function* (flags: DbDiffFlags) { }), ); } + if ( + Option.isSome(flags.useMigra) || + Option.isSome(flags.usePgAdmin) || + Option.isSome(flags.usePgSchema) + ) { + yield* stackRejectNativeDockerDiffEngine; + } // Config is read lazily per path, not unconditionally up front: reading the base config // before the ref is known would validate fields a `[remotes.]` block overrides, which @@ -253,13 +271,41 @@ export const dbDiff = Effect.fn("db.diff")(function* (flags: DbDiffFlags) { Effect.gen(function* () { switch (classifyExplicitRef(ref)) { case "local": { - const connection = { - host: getHostname(), - port: cfg.port, - user: "postgres", - password: cfg.password, - database: "postgres", - }; + const backend = yield* currentStackBackend; + if (backend.kind !== "stack") { + const connection = { + host: getHostname(), + port: cfg.port, + user: "postgres", + password: cfg.password, + database: "postgres", + }; + return { + kind: "database", + ref: toPostgresURL(connection), + connection, + connectOptions: { isLocal: true, dnsResolver }, + } satisfies PgDeltaDatabaseEndpoint; + } + if (Option.isNone(stackApi)) { + return yield* Effect.fail( + new DbDiffDbNotRunningError({ + message: "supabase start is not running.", + }), + ); + } + const connection = yield* stackLocalDatabaseConn.pipe( + Effect.provideService(CommandSettings, cliSettings), + Effect.provideService(StackApi, stackApi.value), + Effect.mapError( + (cause) => + new DbDiffDbNotRunningError({ + message: cause.message, + daemonDown: cause.daemonDown, + suggestion: cause.suggestion, + }), + ), + ); return { kind: "database", ref: toPostgresURL(connection), @@ -495,11 +541,13 @@ export const dbDiff = Effect.fn("db.diff")(function* (flags: DbDiffFlags) { // Engine resolution: the pg-delta env/config/flag gate, read from the // (possibly remote-merged) config. - const pgDeltaDefault = shouldUsePgDelta({ - configEnabled: cfg.pgDelta.enabled, - usePgDeltaFlag: Option.getOrElse(flags.usePgDelta, () => false), - envEnabled: parseBoolEnv(cfg.envLookup("SUPABASE_EXPERIMENTAL_PG_DELTA")), - }); + const pgDeltaDefault = + (yield* currentStackBackend).kind === "stack" || + shouldUsePgDelta({ + configEnabled: cfg.pgDelta.enabled, + usePgDeltaFlag: Option.getOrElse(flags.usePgDelta, () => false), + envEnabled: parseBoolEnv(cfg.envLookup("SUPABASE_EXPERIMENTAL_PG_DELTA")), + }); const useDelta = resolveDiffEngine({ useMigraChanged: Option.isSome(flags.useMigra), usePgAdmin, @@ -521,7 +569,10 @@ export const dbDiff = Effect.fn("db.diff")(function* (flags: DbDiffFlags) { // branch's own "Creating shadow database..." banner announces, so every call site emits its // banner first and only then invokes this. const resolveShadowRunInput = Effect.fnUntraced(function* () { - const resolvedShadowImage = yield* localInputs.resolvePostgresImage; + const stackBackend = (yield* currentStackBackend).kind === "stack"; + const resolvedShadowImage = stackBackend + ? "stack-ephemeral" + : yield* localInputs.resolvePostgresImage; return shadowRunInputFromLocalContainerInputs( localInputs, resolvedShadowImage, @@ -622,63 +673,72 @@ export const dbDiff = Effect.fn("db.diff")(function* (flags: DbDiffFlags) { schemaPaths: cfg.schemaPathPatterns, pgDelta: cfg.pgDelta, }; + const runDiff = ( + shadow: Pick & { + readonly sourceUrl: string; + readonly targetUrlOverride?: string; + }, + ) => + Effect.gen(function* () { + const target = shadow.targetUrlOverride ?? targetUrl; + yield* output.raw( + flags.schema.length > 0 + ? `Diffing schemas: ${flags.schema.join(",")}\n` + : "Diffing schemas...\n", + "stderr", + ); + if (useDelta) { + const result = yield* pgDelta.diffDatabase({ + context: ctx, + source: { + kind: "database", + ref: shadow.sourceUrl, + connectOptions: { isLocal: true, dnsResolver: "native" }, + }, + target: { + kind: "database", + ref: target, + ...(shadow.targetUrlOverride === undefined ? { connection: resolved.conn } : {}), + connectOptions: { + isLocal: shadow.targetUrlOverride !== undefined || resolved.isLocal, + dnsResolver, + }, + }, + schema: flags.schema, + formatOptions, + debug: isPgDeltaDebugEnabled(), + strictCoverage: flags.strictCoverage, + }); + return { sql: result.sql, files: result.files, hazards: result.hazards }; + } + const sql = yield* diffMigra(ctx, { + source: shadow.sourceUrl, + target, + schema: flags.schema, + connectOptions: { isLocal: resolved.isLocal, dnsResolver }, + }); + return { sql, files: undefined }; + }); // `withShadowDatabase` (`shadow-cache.ts`) owns the interrupt-safe lifecycle and the // cache seam — a plain create/remove pair when `SUPABASE_SHADOW_CACHE` is explicitly // disabled (the cache is on by default). The key's webhooks policy must mirror what // `prepareShadowSource` selects for this mode (legacy migrate forces `pg_net` on, // next follows config), or the two engines could restore each other's tars. - diffResult = yield* withShadowDatabase( - spawner, - shadowInput, - (handle) => - Effect.gen(function* () { - const shadow = yield* prepareShadowSource(spawner, handle, shadowInput); - const target = shadow.targetUrlOverride ?? targetUrl; - yield* output.raw( - flags.schema.length > 0 - ? `Diffing schemas: ${flags.schema.join(",")}\n` - : "Diffing schemas...\n", - "stderr", - ); - if (useDelta) { - const result = yield* pgDelta.diffDatabase({ - context: ctx, - source: { - kind: "database", - ref: shadow.sourceUrl, - connectOptions: { isLocal: true, dnsResolver: "native" }, - }, - target: { - kind: "database", - ref: target, - ...(shadow.targetUrlOverride === undefined ? { connection: resolved.conn } : {}), - connectOptions: { - isLocal: shadow.targetUrlOverride !== undefined || resolved.isLocal, - dnsResolver, - }, - }, - schema: flags.schema, - formatOptions, - debug: isPgDeltaDebugEnabled(), - strictCoverage: flags.strictCoverage, - }); - // Keep the per-unit plan files so a multi-unit plan can be written as one - // migration file each; `sql` stays the flattened join for stdout review + - // machine payloads. - return { sql: result.sql, files: result.files, hazards: result.hazards }; - } - const sql = yield* diffMigra(ctx, { - source: shadow.sourceUrl, - target, - schema: flags.schema, - connectOptions: { isLocal: resolved.isLocal, dnsResolver }, - }); - // The migra engine has no execution-aware plan units, so it always writes a - // single migration file. - return { sql, files: undefined }; - }), - { webhooks: migrationMode === "pgdelta-next" ? "config" : "enabled" }, - ); + const stackBackend = (yield* currentStackBackend).kind === "stack"; + diffResult = stackBackend + ? yield* stackWithShadowDatabase(shadowInput, (handle) => + stackPrepareShadowSource(handle, shadowInput).pipe(Effect.flatMap(runDiff)), + ) + : yield* withShadowDatabase( + spawner, + shadowInput, + (handle) => + Effect.gen(function* () { + const shadow = yield* prepareShadowSource(spawner, handle, shadowInput); + return yield* runDiff(shadow); + }), + { webhooks: migrationMode === "pgdelta-next" ? "config" : "enabled" }, + ); } const out = diffResult.sql; diff --git a/apps/cli/src/commands/db/diff/diff.integration.test.ts b/apps/cli/src/commands/db/diff/diff.integration.test.ts index 363299b6a4..3bdbee47bd 100644 --- a/apps/cli/src/commands/db/diff/diff.integration.test.ts +++ b/apps/cli/src/commands/db/diff/diff.integration.test.ts @@ -2,7 +2,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from import { basename, join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, Fiber, Layer, Option } from "effect"; +import { Cause, Effect, Exit, Fiber, Layer, Option } from "effect"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; @@ -63,6 +63,8 @@ import { } from "../shared/pgdelta-engine.service.ts"; import type { DbDiffFlags } from "./diff.command.ts"; import { dbDiff } from "./diff.handler.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; +import { StackNativeEngineError } from "../../../command-internal/stack-local-database.ts"; import { PGADMIN_DESKTOP_NOTE_PREFIX, PGADMIN_DIFF_HEADER } from "./pgadmin-diff.ts"; interface SetupOpts { @@ -1719,6 +1721,28 @@ describe("db diff", () => { }).pipe(Effect.provide(s.layer)); }); + it.effect("rejects --use-migra on the stack backend", () => { + const s = setup(tmp.current); + return Effect.gen(function* () { + const exit = yield* dbDiff(flags({ useMigra: Option.some(true) })).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + expect(error).toBeInstanceOf(StackNativeEngineError); + }).pipe(Effect.provide(Layer.mergeAll(s.layer, stackBackendLayer("stack")))); + }); + + it.effect("rejects --use-migra=false on the stack backend", () => { + const s = setup(tmp.current); + return Effect.gen(function* () { + const exit = yield* dbDiff(flags({ useMigra: Option.some(false) })).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + expect(error).toBeInstanceOf(StackNativeEngineError); + }).pipe(Effect.provide(Layer.mergeAll(s.layer, stackBackendLayer("stack")))); + }); + it.effect("fails on target mutex (--linked with --local)", () => { const s = setup(tmp.current); return Effect.gen(function* () { diff --git a/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md b/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md index 1f36cac1c4..7df2ed6768 100644 --- a/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md @@ -1,7 +1,8 @@ # `supabase db dump` Native TypeScript port (`dump.handler.ts`). Streams a `pg_dump`/`pg_dumpall` -script run inside the local Postgres image to stdout or `--file`. +script run inside the local Postgres image (or PATH `pg_dump`/`pg_dumpall` on +a native stack) to stdout or `--file`. ## Files Read @@ -43,11 +44,11 @@ script run inside the local Postgres image to stdout or `--file`. ## Exit Codes -| Code | Condition | -| ---- | ----------------------------------------------------------------------------------------------------------------------------------- | -| `0` | success | -| `1` | `--use-copy`/`--exclude` without `--data-only`; mutually-exclusive flags; bad `--file` path; connection failure; container exit ≠ 0 | -| `1` | `--project-ref` set with a resolved target other than linked (see Notes / Divergences) | +| Code | Condition | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `0` | success | +| `1` | `--use-copy`/`--exclude` without `--data-only`; mutually-exclusive flags; bad `--file` path; connection failure; container or PATH `pg_dump`/`pg_dumpall` exit ≠ 0 | +| `1` | `--project-ref` set with a resolved target other than linked (see Notes / Divergences) | ## Output diff --git a/apps/cli/src/commands/db/dump/dump.errors.ts b/apps/cli/src/commands/db/dump/dump.errors.ts index f912696e25..387939abda 100644 --- a/apps/cli/src/commands/db/dump/dump.errors.ts +++ b/apps/cli/src/commands/db/dump/dump.errors.ts @@ -44,8 +44,9 @@ export class DbDumpOpenFileError extends Data.TaggedError("DbDumpOpenFileError") } /** - * The pg_dump container exited non-zero; message text - * (`"error running container: exit " + code`) is an established output contract. + * pg_dump exited non-zero. Container dumps keep + * `"error running container: exit " + code`; native PATH dumps use + * `"error running pg_dump: exit " + code` (or `pg_dumpall`). */ export class DbDumpRunError extends Data.TaggedError("DbDumpRunError")<{ readonly message: string; diff --git a/apps/cli/src/commands/db/dump/dump.handler.ts b/apps/cli/src/commands/db/dump/dump.handler.ts index f0ef718103..7c0ca85a9b 100644 --- a/apps/cli/src/commands/db/dump/dump.handler.ts +++ b/apps/cli/src/commands/db/dump/dump.handler.ts @@ -1,4 +1,4 @@ -import { Effect, FileSystem, Option, Path } from "effect"; +import { Effect, FileSystem, Option, Path, Predicate } from "effect"; import { CommandSettings } from "../../../config/command-settings.service.ts"; import { ProjectRefResolver } from "../../../config/project-ref.service.ts"; @@ -17,7 +17,7 @@ import { isIPv6ConnectivityError, } from "../../../command-internal/connect-errors.ts"; import { bold, yellow } from "../../../command-internal/colors.ts"; -import { DnsResolverFlag } from "../../../command-internal/global-flags.ts"; +import { DnsResolverFlag, NetworkIdFlag } from "../../../command-internal/global-flags.ts"; import { RuntimeInfo } from "../../../shared/runtime/runtime-info.service.ts"; import { Tty } from "../../../shared/runtime/tty.service.ts"; import { cobraMutuallyExclusiveErrorMessage } from "../../../shared/cli/cobra-flag-groups.ts"; @@ -35,7 +35,21 @@ import { buildSchemaDumpEnv, expandScript, } from "../../../command-internal/pg-dump.env.ts"; -import { streamPgDump } from "../../../command-internal/pg-dump.run.ts"; +import { + pgDumpClientExitMessage, + streamPgDumpWithClient, +} from "../../../command-internal/pg-dump.run.ts"; +import { + dumpConnForHostClient, + rewriteDumpHostForToolContainer, +} from "../../../command-internal/postgres-client.run.ts"; +import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; +import { + stackProjectDatabaseMajor, + stackRequireProjectRuntime, +} from "../../../command-internal/stack-local-database.ts"; +import { viperEnvStringWithProjectFallback } from "../../../command-internal/viper-env.ts"; +import { DockerRunError } from "../../../command-internal/docker-run.errors.ts"; import { runWithPoolerFallback } from "../shared/pooler-fallback.ts"; import { dumpDataScript, @@ -71,6 +85,7 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const dnsResolver = yield* DnsResolverFlag; + const networkIdFlag = yield* NetworkIdFlag; const tty = yield* Tty; const runtimeInfo = yield* RuntimeInfo; @@ -183,6 +198,38 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { // silently printing a script. const tomlValues = yield* readDbToml(fs, path, cliSettings.workdir, linkedRef); + const backend = yield* currentStackBackend; + const stackRuntime = + backend.kind === "stack" && isLocal ? yield* stackRequireProjectRuntime : undefined; + const useHostClient = stackRuntime?.kind === "native" && runtimeInfo.platform !== "win32"; + const networkId = Option.getOrUndefined(networkIdFlag); + const envNetworkId = viperEnvStringWithProjectFallback("SUPABASE_NETWORK_ID", projectEnv); + const dumpUsesHostNetwork = + backend.kind === "stack" + ? networkId === undefined || networkId.length === 0 + : (networkId === undefined || networkId.length === 0) && envNetworkId.length === 0; + const dumpConn = useHostClient + ? dumpConnForHostClient(conn) + : backend.kind === "stack" && isLocal + ? { + ...conn, + host: rewriteDumpHostForToolContainer(conn.host, { + platform: runtimeInfo.platform, + usesHostNetwork: dumpUsesHostNetwork, + }), + } + : conn; + const serverMajor = + backend.kind === "stack" && isLocal ? yield* stackProjectDatabaseMajor : undefined; + const dumpMajor = serverMajor ?? tomlValues.majorVersion; + const dumpClient = useHostClient + ? { + kind: "host" as const, + command: roleOnly ? ("pg_dumpall" as const) : ("pg_dump" as const), + expectedMajor: dumpMajor, + } + : { kind: "container" as const }; + // 4. Pick the mode-specific script + env. --schema/-s and --exclude/-x arrive here // already CSV-parsed by `parseSchemaFlags`. const opt = { @@ -206,7 +253,7 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { script: dumpSchemaScript, buildEnv: buildSchemaDumpEnv, } as const); - const modeEnv = mode.buildEnv(conn, opt); + const modeEnv = mode.buildEnv(dumpConn, opt); // Keys off `path.length > 0`, not flag presence: `--file ""` means stdout, no // file opened. @@ -231,7 +278,7 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { fs, path, cliSettings.workdir, - tomlValues.majorVersion, + dumpMajor, Option.getOrUndefined(tomlValues.orioledbVersion), ); @@ -278,13 +325,14 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { const file = yield* fs .open(resolvedFile.value, { flag: "a" }) .pipe(Effect.mapError(toOpenFileError)); - return yield* streamPgDump({ + return yield* streamPgDumpWithClient({ image, script: mode.script, env, onStdout: (chunk) => file.writeAll(chunk).pipe(Effect.mapError(toOpenFileError)), projectEnvValues: projectEnv, + client: dumpClient, }); }), ), @@ -293,7 +341,7 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { : // stdout: write each chunk straight to stdout (binary-safe, no decode). // On a pooler retry the partial first-attempt bytes are left on // stdout (a pipe can't be rewound); streaming matches that. - streamPgDump({ + streamPgDumpWithClient({ image, script: mode.script, env, @@ -307,31 +355,49 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { }) : (chunk) => output.rawBytes(chunk), projectEnvValues: projectEnv, + client: dumpClient, }); // 7b. IPv6 → IPv4-pooler retry, shared with `db pull`: a linked dump can reach the // direct host from the CLI process yet fail inside the container on an // IPv6-only Docker network. Falls back to `None` on any resolution error so the // original pg_dump failure surfaces instead of a fallback-setup error. - const result = yield* runWithPoolerFallback({ - result: yield* runContainer(modeEnv), - connType, - host: conn.host, - isLocal, - projectHost: cliSettings.projectHost, - resolvePooler: () => - resolver - .resolvePoolerFallback({ - dbUrl: flags.dbUrl, - connType: "linked", - dnsResolver, - password: flags.password, - linkedProjectRef: flags.projectRef, - }) - .pipe(Effect.orElseSucceed(() => Option.none())), - runWithConn: (c) => runContainer(mode.buildEnv(c, opt)), - reprintOnRetry: output.raw(`Dumping ${mode.verb} from ${db} database...\n`, "stderr"), - }); + const result = yield* runContainer(modeEnv).pipe( + Effect.flatMap((dumped) => + runWithPoolerFallback({ + result: dumped, + connType, + host: conn.host, + isLocal, + projectHost: cliSettings.projectHost, + resolvePooler: () => + resolver + .resolvePoolerFallback({ + dbUrl: flags.dbUrl, + connType: "linked", + dnsResolver, + password: flags.password, + linkedProjectRef: flags.projectRef, + }) + .pipe(Effect.orElseSucceed(() => Option.none())), + runWithConn: (c) => runContainer(mode.buildEnv(c, opt)), + reprintOnRetry: output.raw(`Dumping ${mode.verb} from ${db} database...\n`, "stderr"), + }), + ), + Effect.catchIf( + (error): error is DockerRunError => + Predicate.isTagged(error, "DockerRunError") && + stackRuntime?.kind === "native" && + runtimeInfo.platform === "win32", + (error) => + Effect.fail( + new DbDumpRunError({ + message: error.message, + suggestion: "Install Docker Desktop (or Git Bash) to dump a native stack on Windows.", + }), + ), + ), + ); // 8. The dump has already been streamed to the destination by `runContainer` // (to `--file` or stdout) as pg_dump produced it. @@ -342,7 +408,7 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { if (result.exitCode !== 0) { return yield* Effect.fail( new DbDumpRunError({ - message: `error running container: exit ${result.exitCode}`, + message: pgDumpClientExitMessage(dumpClient, result.exitCode), ...(isIPv6ConnectivityError(result.stderr) ? { suggestion: ipv6Suggestion() } : {}), }), ); diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index 85cbd4ed99..b768c318f8 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -4,7 +4,8 @@ import process from "node:process"; import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option } from "effect"; +import { Cause, Effect, Exit, Layer, Option, Sink, Stream } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { mockOutput, mockTty, processEnvLayer } from "../../../../tests/helpers/mocks.ts"; import { @@ -34,6 +35,9 @@ import { DockerRunError } from "../../../command-internal/docker-run.errors.ts"; import { DockerRun, type DockerRunOpts } from "../../../command-internal/docker-run.service.ts"; import type { DbDumpFlags } from "./dump.command.ts"; import { dbDump } from "./dump.handler.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; +import { StackApi } from "../../../command-internal/stack-api.ts"; +import { StackIdSchema, type EffectStack } from "@supabase/stack/effect"; const LOCAL_CONN: PgConnInput = { host: "127.0.0.1", @@ -972,4 +976,302 @@ describe("db dump integration", () => { }).pipe(Effect.provide(layer)); }); } + + const DUMP_STACK_ID = StackIdSchema.make("d".repeat(64)); + const unusedDump = () => Effect.die("unused"); + const unusedDumpEffect = Effect.die("unused"); + const dumpStackApi = ( + runtime: { kind: "native" } | { kind: "container"; engine: "docker" }, + databaseVersion = "17.6.1", + ) => { + const stack: EffectStack = { + id: DUMP_STACK_ID, + status: Effect.succeed({ + id: DUMP_STACK_ID, + lifecycle: "running", + desiredLifecycle: "running", + runtime, + endpoints: {}, + versions: { database: databaseVersion }, + capabilities: [], + artifacts: [], + }), + credentials: unusedDumpEffect, + prepare: unusedDump, + start: unusedDump, + stop: unusedDumpEffect, + destroy: unusedDumpEffect, + resetDatabase: unusedDumpEffect, + logs: unusedDump, + followLogs: () => Stream.empty, + }; + return Layer.succeed(StackApi, { + createStack: unusedDump, + findStack: () => + Effect.succeed( + Option.some({ + id: DUMP_STACK_ID, + projectRoot: "/work/project", + name: "default", + branchContext: "main", + runtime, + desiredLifecycle: "running", + }), + ), + discoverStacks: unusedDump, + openStack: () => Effect.succeed(stack), + inspectStack: unusedDump, + }); + }; + + it.live( + "dump --local on the stack backend fails instead of using Docker when no stack exists", + () => { + const { layer, docker } = setup({ isLocal: true, stdout: "-- schema\n" }); + return Effect.gen(function* () { + const exit = yield* Effect.exit(dbDump(flags({ local: Option.some(true) }))); + expect(Exit.isFailure(exit)).toBe(true); + expect(failMessage(exit)).toContain("Could not determine the stack runtime"); + expect(docker.lastOpts).toBeUndefined(); + }).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + Layer.succeed(StackApi, { + createStack: unusedDump, + findStack: () => Effect.succeed(Option.none()), + discoverStacks: unusedDump, + openStack: unusedDump, + inspectStack: unusedDump, + }), + ), + ), + ); + }, + ); + + it.live("dump --local on a docker stack never uses PGHOST=db", () => { + const { layer, docker } = setup({ + isLocal: true, + stdout: "-- schema\n", + platform: "darwin", + }); + return Effect.gen(function* () { + yield* dbDump(flags({ local: Option.some(true) })); + expect(docker.lastOpts?.env["PGHOST"]).toBe("host.docker.internal"); + expect(docker.lastOpts?.env["PGHOST"]).not.toBe("db"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + dumpStackApi({ kind: "container", engine: "docker" }), + ), + ), + ); + }); + + it.live("dump --local on a docker stack ignores compose SUPABASE_NETWORK_ID", () => { + const { layer, docker } = setup({ + isLocal: true, + stdout: "-- schema\n", + platform: "darwin", + env: { SUPABASE_NETWORK_ID: "supabase_network_test" }, + }); + return Effect.gen(function* () { + yield* dbDump(flags({ local: Option.some(true) })); + expect(docker.lastOpts?.network).toEqual({ _tag: "host" }); + expect(docker.lastOpts?.env["PGHOST"]).toBe("host.docker.internal"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + dumpStackApi({ kind: "container", engine: "docker" }), + ), + ), + ); + }); + + it.live("dump --local on a native stack uses PATH pg_dump, not a tool container", () => { + mkdirSync(join(tmp.current, "supabase"), { recursive: true }); + writeFileSync( + join(tmp.current, "supabase", "config.toml"), + 'project_id = "test"\n[db]\nmajor_version = 17\n', + ); + const spawned: Array = []; + const spawner = ChildProcessSpawner.make((command) => + Effect.sync(() => { + const name = command._tag === "StandardCommand" ? command.command : ""; + spawned.push(name); + const stdoutText = name === "pg_dump" ? "pg_dump (PostgreSQL) 17.4\n" : "-- schema\n"; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout: Stream.fromIterable([new TextEncoder().encode(stdoutText)]), + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + const { layer, docker, out } = setup({ + isLocal: true, + workdir: tmp.current, + }); + return Effect.gen(function* () { + yield* dbDump(flags({ local: Option.some(true) })); + expect(docker.lastOpts).toBeUndefined(); + expect(spawned).toContain("pg_dump"); + expect(spawned).toContain("bash"); + expect(out.stdoutText).toContain("-- schema"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + dumpStackApi({ kind: "native" }), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), + ), + ), + ); + }); + + it.live("dump --local on a native stack reports PATH pg_dump exit, not a container", () => { + mkdirSync(join(tmp.current, "supabase"), { recursive: true }); + writeFileSync( + join(tmp.current, "supabase", "config.toml"), + 'project_id = "test"\n[db]\nmajor_version = 17\n', + ); + const spawner = ChildProcessSpawner.make((command) => + Effect.sync(() => { + const name = command._tag === "StandardCommand" ? command.command : ""; + const stdoutText = name === "pg_dump" ? "pg_dump (PostgreSQL) 17.4\n" : "partial\n"; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout: Stream.fromIterable([new TextEncoder().encode(stdoutText)]), + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(name === "bash" ? 1 : 0)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + const { layer, docker } = setup({ + isLocal: true, + workdir: tmp.current, + }); + return Effect.gen(function* () { + const exit = yield* dbDump(flags({ local: Option.some(true) })).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failMessage(exit)).toBe("error running pg_dump: exit 1"); + expect(docker.lastOpts).toBeUndefined(); + }).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + dumpStackApi({ kind: "native" }), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), + ), + ), + ); + }); + + it.live("dump --local on a Windows native stack uses a Docker pg_dump client", () => { + const { layer, docker } = setup({ + isLocal: true, + stdout: "-- schema\n", + platform: "win32", + }); + return Effect.gen(function* () { + yield* dbDump(flags({ local: Option.some(true) })); + expect(docker.lastOpts?.env["PGHOST"]).toBe("host.docker.internal"); + expect(docker.lastOpts?.network).toEqual({ _tag: "host" }); + }).pipe( + Effect.provide( + Layer.mergeAll(layer, stackBackendLayer("stack"), dumpStackApi({ kind: "native" })), + ), + ); + }); + + it.live( + "dump --local on a Windows native stack asks for Docker Desktop when Docker is missing", + () => { + const { layer } = setup({ + isLocal: true, + platform: "win32", + runFails: true, + }); + return Effect.gen(function* () { + const exit = yield* dbDump(flags({ local: Option.some(true) })).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failMessage(exit)).toContain("failed to run docker"); + expect(failSuggestion(exit)).toContain("Docker Desktop"); + }).pipe( + Effect.provide( + Layer.mergeAll(layer, stackBackendLayer("stack"), dumpStackApi({ kind: "native" })), + ), + ); + }, + ); + + it.live( + "dump --local on the stack backend uses status().versions.database for the client major", + () => { + mkdirSync(join(tmp.current, "supabase"), { recursive: true }); + writeFileSync( + join(tmp.current, "supabase", "config.toml"), + 'project_id = "test"\n[db]\nmajor_version = 17\n', + ); + const spawner = ChildProcessSpawner.make((command) => + Effect.sync(() => { + const name = command._tag === "StandardCommand" ? command.command : ""; + const stdoutText = name === "pg_dump" ? "pg_dump (PostgreSQL) 17.4\n" : "-- schema\n"; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout: Stream.fromIterable([new TextEncoder().encode(stdoutText)]), + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + const { layer } = setup({ + isLocal: true, + workdir: tmp.current, + }); + return Effect.gen(function* () { + const exit = yield* dbDump(flags({ local: Option.some(true) })).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failMessage(exit)).toContain("does not match stack Postgres 16"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + dumpStackApi({ kind: "native" }, "16.6.1"), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), + ), + ), + ); + }, + ); }); diff --git a/apps/cli/src/commands/db/dump/dump.layers.ts b/apps/cli/src/commands/db/dump/dump.layers.ts index d97f74d197..6181526246 100644 --- a/apps/cli/src/commands/db/dump/dump.layers.ts +++ b/apps/cli/src/commands/db/dump/dump.layers.ts @@ -13,7 +13,7 @@ import { identityStitchLayer } from "../../../command-internal/identity-stitch.t import { linkedProjectCacheLayer } from "../../../telemetry/linked-project-cache.layer.ts"; import { telemetryStateLayer } from "../../../telemetry/telemetry-state.layer.ts"; import { commandRuntimeLayer } from "../../../shared/runtime/command-runtime.layer.ts"; - +import { stackApiLayer } from "../../../command-internal/stack-api.ts"; /** * Runtime layer for `supabase db dump`. * @@ -74,5 +74,7 @@ export const dbDumpRuntimeLayer = Layer.mergeAll( linkedProjectCache, identityStitchLayer, telemetryStateLayer, + // Exposed so native-engine dump can read `runtime.kind` and pick PATH pg_dump. + stackApiLayer, commandRuntimeLayer(["db", "dump"]), ); diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index d19a7e9eef..fd7a19741f 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -3,7 +3,7 @@ import { dirname, join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option, PlatformError, Sink, Stream } from "effect"; +import { Cause, Effect, Exit, Layer, Option, PlatformError, Sink, Stream, Redacted } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; @@ -42,6 +42,10 @@ import { } from "../../../command-internal/global-flags.ts"; import type { OutputFormat } from "../../../shared/output/types.ts"; import { dockerRunLayer } from "../../../command-internal/docker-run.layer.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; +import { StackApi } from "../../../command-internal/stack-api.ts"; +import { recordingStackCatalogSetup } from "../../../command-internal/stack-catalog-setup.ts"; +import { CAPABILITY_NAMES, StackIdSchema, type EffectStack } from "@supabase/stack/effect"; import { DbConfigResolver } from "../../../command-internal/db-config.service.ts"; import type { DbConfigFlags, ResolvedDbConfig } from "../../../command-internal/db-config.types.ts"; import { DbConfigConnectTempRoleError } from "../../../command-internal/db-config.errors.ts"; @@ -389,6 +393,88 @@ const alwaysReadyHttpClientLayer = Layer.succeed( ), ); +const RESET_STACK_ID = StackIdSchema.make("c".repeat(64)); + +function mockResetStackApi(opts: { readonly workdir: string; readonly ready: boolean }) { + let resetCalls = 0; + const unused = () => Effect.die("unused"); + const unusedEffect = Effect.die("unused"); + const stack: EffectStack = { + id: RESET_STACK_ID, + status: Effect.succeed({ + id: RESET_STACK_ID, + lifecycle: opts.ready ? "running" : "stopped", + desiredLifecycle: opts.ready ? "running" : "stopped", + runtime: { kind: "native" }, + endpoints: {}, + versions: {}, + capabilities: CAPABILITY_NAMES.map((name) => ({ + name, + activation: name === "database" ? "eager" : "lazy", + state: name === "database" && opts.ready ? "ready" : "stopped", + })), + artifacts: [], + }), + credentials: Effect.succeed({ + database: { + url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:54329/postgres"), + password: Redacted.make("postgres"), + }, + api: { + publishableKey: "anon", + secretKey: Redacted.make("service"), + anonJwt: "anon", + serviceRoleJwt: Redacted.make("service"), + }, + }), + prepare: unused, + start: unused, + stop: unusedEffect, + destroy: unusedEffect, + resetDatabase: Effect.sync(() => { + resetCalls++; + return { + id: RESET_STACK_ID, + lifecycle: "running" as const, + desiredLifecycle: "running" as const, + runtime: { kind: "native" as const }, + endpoints: {}, + versions: {}, + capabilities: CAPABILITY_NAMES.map((name) => ({ + name, + activation: name === "database" ? ("eager" as const) : ("lazy" as const), + state: name === "database" ? ("ready" as const) : ("dormant" as const), + })), + artifacts: [], + }; + }), + logs: unused, + followLogs: () => Stream.empty, + }; + return { + layer: Layer.succeed(StackApi, { + createStack: unused, + findStack: () => + Effect.succeed( + Option.some({ + id: RESET_STACK_ID, + projectRoot: opts.workdir, + name: "default", + branchContext: "main", + runtime: { kind: "native" as const }, + desiredLifecycle: "running" as const, + }), + ), + discoverStacks: unused, + openStack: () => Effect.succeed(stack), + inspectStack: unused, + }), + get resetCalls() { + return resetCalls; + }, + }; +} + function setup( workdir: string, opts: { @@ -417,6 +503,8 @@ function setup( // Simulates an unlinked workdir: `loadProjectRef` fails with `ProjectRefNotLinkedError` // absent an explicit `--project-ref` flag, instead of falling back to `opts.ref`. linkedFails?: boolean; + stackBackend?: boolean; + stackDatabaseReady?: boolean; }, ) { if (opts.toml !== undefined) { @@ -444,6 +532,14 @@ function setup( }); const route = opts.route ?? defaultLocalResetRoute(opts.routeOpts); const child = mockContainerCliSpawner(route); + const stackApi = mockResetStackApi({ + workdir, + ready: opts.stackDatabaseReady !== false, + }); + const catalog = + opts.stackBackend === true + ? recordingStackCatalogSetup((input) => input.target.kind) + : undefined; const layer = Layer.mergeAll( out.layer, conn.layer, @@ -485,6 +581,9 @@ function setup( Layer.succeed(DebugFlag, opts.debug ?? false), telemetry.layer, linkedCache.layer, + ...(opts.stackBackend === true && catalog !== undefined + ? [stackBackendLayer("stack"), stackApi.layer, catalog.layer] + : []), ); return { layer, @@ -494,6 +593,8 @@ function setup( linkedCache, resolver, child, + stackApi, + catalogApplied: catalog?.applied ?? [], }; } @@ -666,6 +767,39 @@ describe("db reset", () => { }, ); + it.live("resets the stack database without Compose volume recreate", () => { + const { layer, child, stackApi, catalogApplied } = setup(tmp.current, { + toml: 'project_id = "test"\n', + args: ["db", "reset", "--local"], + isLocal: true, + stackBackend: true, + }); + return Effect.gen(function* () { + yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer)); + expect(stackApi.resetCalls).toBe(1); + expect(catalogApplied).toEqual(["live"]); + expect(child.spawned.some((s) => s.args[0] === "container" && s.args[1] === "rm")).toBe( + false, + ); + }); + }); + + it.live("fails --local reset when the stack database is not running", () => { + const { layer, stackApi } = setup(tmp.current, { + toml: 'project_id = "test"\n', + args: ["db", "reset", "--local"], + isLocal: true, + stackBackend: true, + stackDatabaseReady: false, + }); + return Effect.gen(function* () { + const exit = yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer), Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) expect(JSON.stringify(exit.cause)).toContain("is not running."); + expect(stackApi.resetCalls).toBe(0); + }); + }); + it.live( "fails a local reset before the destructive recreate on a malformed config.toml", () => { diff --git a/apps/cli/src/commands/db/reset/reset.layers.ts b/apps/cli/src/commands/db/reset/reset.layers.ts index 384615d627..ab0007dcf4 100644 --- a/apps/cli/src/commands/db/reset/reset.layers.ts +++ b/apps/cli/src/commands/db/reset/reset.layers.ts @@ -15,6 +15,8 @@ import { stdinLayer } from "../../../shared/runtime/stdin.layer.ts"; import { identityStitchLayer } from "../../../command-internal/identity-stitch.ts"; import { linkedProjectCacheLayer } from "../../../telemetry/linked-project-cache.layer.ts"; import { telemetryStateLayer } from "../../../telemetry/telemetry-state.layer.ts"; +import { stackApiLayer } from "../../../command-internal/stack-api.ts"; +import { stackCatalogSetupLayer } from "../../../command-internal/stack-catalog-setup.ts"; /** * Runtime layer for `supabase db reset`: the Postgres connection, the db-config resolver, @@ -78,5 +80,8 @@ export const dbResetRuntimeLayer = Layer.mergeAll( dockerRunLayer, // Backs `isLocalDbRunning`'s direct Engine-API probe (+ its `--debug` trace). localDockerEngineLayer.pipe(Layer.provide(debugLoggerLayer)), + // Exposed so `db reset --local` can open the project stack and call `resetDatabase`. + stackApiLayer, + stackCatalogSetupLayer, commandRuntimeLayer(["db", "reset"]), ); diff --git a/apps/cli/src/commands/db/schema/declarative/declarative.smart-target.ts b/apps/cli/src/commands/db/schema/declarative/declarative.smart-target.ts index 034a172138..ddcac98275 100644 --- a/apps/cli/src/commands/db/schema/declarative/declarative.smart-target.ts +++ b/apps/cli/src/commands/db/schema/declarative/declarative.smart-target.ts @@ -8,6 +8,7 @@ import { promptYesNo } from "../../../../command-internal/prompt-yes-no.ts"; import { Output } from "../../../../shared/output/output.service.ts"; import { resetLocalDatabase } from "../../../../command-internal/db-bootstrap/reset-local-database.ts"; import { PROJECT_REF_PATTERN } from "../../../../config/project-ref.service.ts"; +import { currentStackBackend } from "../../../../command-internal/stack-backend.ts"; import { DbConfigResolver } from "../../../../command-internal/db-config.service.ts"; import { loadProjectEnv } from "../../../../command-internal/db-config.toml-read.ts"; import { @@ -54,7 +55,7 @@ const localConnection = (local: LocalConn) => ({ database: "postgres", }); -export const localEndpoint = ( +const localEndpoint = ( local: LocalConn, dnsResolver: "native" | "https", ): PgDeltaDatabaseEndpoint => { @@ -67,6 +68,27 @@ export const localEndpoint = ( }; }; +/** Local target URL: stack credentials when the stack backend is on, else config.toml `[db]`. */ +export const resolveLocalTargetEndpoint = Effect.fnUntraced(function* ( + local: LocalConn, + dnsResolver: "native" | "https", +) { + const backend = yield* currentStackBackend; + if (backend.kind !== "stack") return localEndpoint(local, dnsResolver); + const resolver = yield* DbConfigResolver; + const resolved = yield* resolver.resolve({ + dbUrl: Option.none(), + connType: "local", + dnsResolver, + }); + return { + kind: "database", + ref: toPostgresURL(resolved.conn), + connection: resolved.conn, + connectOptions: { isLocal: true, dnsResolver }, + } satisfies PgDeltaDatabaseEndpoint; +}); + /** Resolves a remote target without discarding TLS and connection options. */ export const resolveRemoteEndpoint = Effect.fnUntraced(function* (flags: SmartTargetFlags) { const resolver = yield* DbConfigResolver; @@ -104,7 +126,7 @@ export const resolveSmartTargetEndpoint = Effect.fnUntraced(function* ( // No migrations: generate from local, starting a stopped stack first. yield* beforeLocalTarget; yield* (yield* DeclarativeSeam).ensureLocalDatabaseStarted(); - return localEndpoint(local, yield* DnsResolverFlag); + return yield* resolveLocalTargetEndpoint(local, yield* DnsResolverFlag); } const output = yield* Output; @@ -189,5 +211,5 @@ export const resolveSmartTargetEndpoint = Effect.fnUntraced(function* ( ), ); } - return localEndpoint(local, yield* DnsResolverFlag); + return yield* resolveLocalTargetEndpoint(local, yield* DnsResolverFlag); }); diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.handler.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.handler.ts index 249cd9df18..e3f7436891 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.handler.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.handler.ts @@ -44,7 +44,7 @@ import { import type { DbSchemaDeclarativeGenerateFlags } from "./generate.command.ts"; import { type LocalConn, - localEndpoint, + resolveLocalTargetEndpoint, resolveRemoteEndpoint, resolveSmartTargetEndpoint, } from "../declarative.smart-target.ts"; @@ -178,7 +178,7 @@ export const dbSchemaDeclarativeGenerate = Effect.fn("db.schema.declarative.gene if (Option.getOrElse(flags.local, () => false)) { yield* seam.ensureLocalDatabaseStarted(); } - target = localEndpoint(local, dnsResolver); + target = yield* resolveLocalTargetEndpoint(local, dnsResolver); } else { target = yield* resolveRemoteEndpoint(flags); } diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index 862855f61e..bf68e2a5fb 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -41,6 +41,7 @@ import { GoProxy } from "../../../../../command-internal/go-proxy.service.ts"; import { CommandPlatformApi } from "../../../../../auth/command-platform-api.service.ts"; import { CommandPlatformApiFactory } from "../../../../../auth/command-platform-api-factory.service.ts"; import { dockerRunLayer } from "../../../../../command-internal/docker-run.layer.ts"; +import { stackBackendLayer } from "../../../../../command-internal/stack-backend.ts"; import { DbConfigResolver } from "../../../../../command-internal/db-config.service.ts"; import { type DbSession, @@ -71,6 +72,7 @@ interface SetupOpts { /** Makes the engine's `exportDeclarativeSchema` fail after recording the call. */ exportFails?: boolean; staleLocalImage?: boolean; + stackBackend?: boolean; } /** What the handler handed the engine for one `exportDeclarativeSchema` call. */ @@ -171,6 +173,18 @@ function setup(workdir: string, opts: SetupOpts = {}) { const resolver = Layer.succeed(DbConfigResolver, { resolve: (flags) => { resolverCalls.push(flags); + if (flags.connType === "local") { + return Effect.succeed({ + conn: { + host: "127.0.0.1", + port: 54329, + user: "postgres", + password: "stack-secret", + database: "postgres", + }, + isLocal: true, + }); + } return Effect.succeed({ conn: { host: "db.remote", @@ -234,6 +248,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { processControl.layer, alwaysReadyHttpClientLayer, dockerRun, + ...(opts.stackBackend === true ? [stackBackendLayer("stack")] : []), ); return { layer, @@ -435,6 +450,20 @@ describe("db schema declarative generate integration", () => { }).pipe(Effect.provide(s.layer)); }); + it.effect( + "explicit --local on the stack backend exports from stack credentials, not toml [db].port", + () => { + const s = setup(tmp.current, { experimental: true, stackBackend: true }); + return Effect.gen(function* () { + yield* dbSchemaDeclarativeGenerate(flags({ local: Option.some(true) })); + expect(s.engineExportCalls[0]!.targetRef).toContain( + "postgresql://postgres:stack-secret@127.0.0.1:54329", + ); + expect(s.engineExportCalls[0]!.targetRef).not.toContain(":54322"); + }).pipe(Effect.provide(s.layer)); + }, + ); + it.effect( "--output-dir writes a complete export relative to the project without activating it", () => { diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.handler.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.handler.ts index f061a8e813..4e574318c4 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.handler.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.handler.ts @@ -10,6 +10,8 @@ import { Output } from "../../../../../shared/output/output.service.ts"; import { Tty } from "../../../../../shared/runtime/tty.service.ts"; import { CommandSettings } from "../../../../../config/command-settings.service.ts"; import { resetLocalDatabase } from "../../../../../command-internal/db-bootstrap/reset-local-database.ts"; +import { stackLocalDatabaseConn } from "../../../../../command-internal/stack-local-database.ts"; +import { currentStackBackend } from "../../../../../command-internal/stack-backend.ts"; import { bold, red, yellow } from "../../../../../command-internal/colors.ts"; import { DbConnection } from "../../../../../command-internal/db-connection.service.ts"; import { getHostname } from "../../../../../command-internal/hostname.ts"; @@ -31,7 +33,10 @@ import { resolvePgDeltaProjectId, } from "../../../../../command-internal/pgdelta.ts"; import { writePgDeltaMigrations } from "../../../shared/pgdelta-migrations.write.ts"; -import { localEndpoint, resolveSmartTargetEndpoint } from "../declarative.smart-target.ts"; +import { + resolveLocalTargetEndpoint, + resolveSmartTargetEndpoint, +} from "../declarative.smart-target.ts"; import { type DebugBundle, collectMigrationsList, @@ -306,7 +311,10 @@ export const dbSchemaDeclarativeSync = Effect.fn("db.schema.declarative.sync")(f } const generated = yield* generateDeclarativeOutput( { ...run, declarativeDir: stagedDir }, - localEndpoint({ port: toml.port, password: toml.password }, dnsResolver), + yield* resolveLocalTargetEndpoint( + { port: toml.port, password: toml.password }, + dnsResolver, + ), ); const written = yield* writeDeclarativeSchemas(fs, path, stagedDir, generated); yield* warnPreservedUnmanagedDeclarativeFiles(stagedDirRel, written); @@ -573,8 +581,26 @@ export const dbSchemaDeclarativeSync = Effect.fn("db.schema.declarative.sync")(f // Step 8: apply the migration to the local database (native). yield* ensureLocalPostgresImageCurrent; + const backend = yield* currentStackBackend; + const applyTarget = + backend.kind === "stack" + ? yield* stackLocalDatabaseConn.pipe( + Effect.mapError( + (error) => new DeclarativeApplyError({ message: error.message, connect: true }), + ), + ) + : { + host: getHostname(), + port: toml.port, + password: toml.password, + }; const applyExit = yield* applyMigrationToLocal( - { port: toml.port, password: toml.password, dnsResolver }, + { + host: applyTarget.host, + port: applyTarget.port, + password: applyTarget.password, + dnsResolver, + }, migrationPaths, ).pipe(Effect.exit); @@ -684,7 +710,7 @@ const declarativeDirHasFiles = Effect.fnUntraced(function* ( /** Connects once and applies the ordered migration files. */ const applyMigrationToLocal = ( - local: { port: number; password: string; dnsResolver: "native" | "https" }, + local: { host: string; port: number; password: string; dnsResolver: "native" | "https" }, migrationPaths: ReadonlyArray, ) => Effect.gen(function* () { @@ -694,9 +720,7 @@ const applyMigrationToLocal = ( const session = yield* dbConnection .connect( { - // Host resolution order: SUPABASE_SERVICES_HOSTNAME → tcp DOCKER_HOST → 127.0.0.1, not - // a hardcoded loopback. - host: getHostname(), + host: local.host, port: local.port, user: "postgres", password: local.password, diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index f8110f21c1..1dda37faa0 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -2,7 +2,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option } from "effect"; +import { Cause, Effect, Exit, Layer, Option, Stream, Redacted } from "effect"; import { stripAnsi } from "../../../../../../tests/helpers/ansi.ts"; import { @@ -39,6 +39,9 @@ import { import { CommandPlatformApi } from "../../../../../auth/command-platform-api.service.ts"; import { CommandPlatformApiFactory } from "../../../../../auth/command-platform-api-factory.service.ts"; import { dockerRunLayer } from "../../../../../command-internal/docker-run.layer.ts"; +import { stackBackendLayer } from "../../../../../command-internal/stack-backend.ts"; +import { StackApi } from "../../../../../command-internal/stack-api.ts"; +import { CAPABILITY_NAMES, StackIdSchema, type EffectStack } from "@supabase/stack/effect"; import { DbConfigResolver } from "../../../../../command-internal/db-config.service.ts"; import { type DbBatchStatement, @@ -79,6 +82,68 @@ interface SetupOpts { renderedFiles?: ReadonlyArray; removals?: PgDeltaRemovalSummary; planErrors?: ReadonlyArray; + stackBackend?: boolean; +} + +const SYNC_STACK_ID = StackIdSchema.make("e".repeat(64)); +const unusedSync = () => Effect.die("unused"); +const unusedSyncEffect = Effect.die("unused"); +const STACK_APPLY_PORT = 54329; + +function syncStackApi(workdir: string, port: number) { + const stack: EffectStack = { + id: SYNC_STACK_ID, + status: Effect.succeed({ + id: SYNC_STACK_ID, + lifecycle: "running", + desiredLifecycle: "running", + runtime: { kind: "native" }, + endpoints: {}, + versions: {}, + capabilities: CAPABILITY_NAMES.map((name) => ({ + name, + activation: name === "database" ? "eager" : "lazy", + state: name === "database" ? "ready" : "dormant", + })), + artifacts: [], + }), + credentials: Effect.succeed({ + database: { + url: Redacted.make(`postgresql://postgres:postgres@127.0.0.1:${port}/postgres`), + password: Redacted.make("postgres"), + }, + api: { + publishableKey: "anon", + secretKey: Redacted.make("service"), + anonJwt: "anon", + serviceRoleJwt: Redacted.make("service"), + }, + }), + prepare: unusedSync, + start: unusedSync, + stop: unusedSyncEffect, + destroy: unusedSyncEffect, + resetDatabase: unusedSyncEffect, + logs: unusedSync, + followLogs: () => Stream.empty, + }; + return Layer.succeed(StackApi, { + createStack: unusedSync, + findStack: () => + Effect.succeed( + Option.some({ + id: SYNC_STACK_ID, + projectRoot: workdir, + name: "default", + branchContext: "main", + runtime: { kind: "native" as const }, + desiredLifecycle: "running", + }), + ), + discoverStacks: unusedSync, + openStack: () => Effect.succeed(stack), + inspectStack: unusedSync, + }); } function setup(workdir: string, opts: SetupOpts = {}) { @@ -119,9 +184,11 @@ function setup(workdir: string, opts: SetupOpts = {}) { // shadow also connects through this fake `DbConnection`, so its SQL must be excluded from // `dbExec`, which every "not yet applied" assertion expects to stay empty until real apply. const SHADOW_PORT = 54320; + const dbConnectPorts: number[] = []; const dbConn = Layer.succeed(DbConnection, { - connect: (cfg: PgConnInput) => - Effect.succeed({ + connect: (cfg: PgConnInput) => { + if (cfg.port !== SHADOW_PORT) dbConnectPorts.push(cfg.port); + return Effect.succeed({ exec: (sql: string) => opts.applyFails === true && sql.startsWith("ALTER") ? Effect.fail({ _tag: "DbExecError", message: "boom" } as never) @@ -155,7 +222,8 @@ function setup(workdir: string, opts: SetupOpts = {}) { extensionExists: () => Effect.succeed(false), copyToCsv: () => Effect.succeed(new Uint8Array()), queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), - }), + }); + }, }); // The no-files bootstrap delegates to the shared smart-target resolver; its // local path never calls `resolve`, but the linked/custom branches would. @@ -265,6 +333,9 @@ function setup(workdir: string, opts: SetupOpts = {}) { processControl.layer, alwaysReadyHttpClientLayer, dockerRun, + ...(opts.stackBackend === true + ? [stackBackendLayer("stack"), syncStackApi(workdir, STACK_APPLY_PORT)] + : []), ); return { layer, @@ -272,6 +343,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { child, dbExec, dbBatches, + dbConnectPorts, cache, telemetry, localPostgresImageChecks, @@ -827,6 +899,20 @@ describe("db schema declarative sync integration", () => { }).pipe(Effect.provide(s.layer)); }); + it.effect("--apply on the stack backend uses stack credentials, not toml.port", () => { + seedDeclarative(tmp.current); + const s = setup(tmp.current, { + experimental: true, + diffSql: "ALTER TABLE a ADD COLUMN b int;\n", + stackBackend: true, + }); + return Effect.gen(function* () { + yield* dbSchemaDeclarativeSync(flags({ apply: Option.some(true) })); + expect(s.dbConnectPorts).toContain(STACK_APPLY_PORT); + expect(s.dbConnectPorts).not.toContain(54322); + }).pipe(Effect.provide(s.layer)); + }); + it.effect("refuses a known implicit-extension load failure under --yes", () => { seedUuidDeclarative(tmp.current); const s = setup(tmp.current, { diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts index ca49141a92..81755650e7 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts @@ -1,4 +1,4 @@ -import { Effect, FileSystem, Layer, Option, Path } from "effect"; +import { Crypto, Effect, FileSystem, Layer, Option, Path } from "effect"; import * as Net from "node:net"; import { CliArgs } from "../../../shared/cli/cli-args.service.ts"; import { @@ -10,6 +10,7 @@ import { import { Output } from "../../../shared/output/output.service.ts"; import { RuntimeInfo } from "../../../shared/runtime/runtime-info.service.ts"; import { DbConnection } from "../../../command-internal/db-connection.service.ts"; +import { CommandSettings } from "../../../config/command-settings.service.ts"; import { DockerRun } from "../../../command-internal/docker-run.service.ts"; import { toPostgresURL } from "../../../command-internal/postgres-url.ts"; import { @@ -48,6 +49,12 @@ import { type PgDeltaNextShadowInput, } from "./pgdelta-next-shadow.service.ts"; import { DeclarativeShadowDbError } from "./pgdelta.errors.ts"; +import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; +import { + stackAcquireShadowDatabase, + stackMigrateShadow, +} from "../../../command-internal/stack-shadow.ts"; +import { stackCatalogSetupLayer } from "../../../command-internal/stack-catalog-setup.ts"; const allocateFreeHostPort = Effect.callback>((resume) => { const server = Net.createServer(); @@ -136,9 +143,11 @@ export const pgDeltaNextShadowLayer = Layer.effect( const docker = yield* DockerRun; const dbConnection = yield* DbConnection; const httpClient = yield* HttpClient.HttpClient; + const crypto = yield* Crypto.Crypto; + const cliSettings = yield* CommandSettings; - const runtimeWith = (outputService: typeof Output.Service) => - Layer.mergeAll( + const runtimeWith = (outputService: typeof Output.Service) => { + const deps = Layer.mergeAll( Layer.succeed(FileSystem.FileSystem, fs), Layer.succeed(Path.Path, path), Layer.succeed(DebugFlag, debugFlag), @@ -150,7 +159,12 @@ export const pgDeltaNextShadowLayer = Layer.effect( Layer.succeed(DockerRun, docker), Layer.succeed(DbConnection, dbConnection), Layer.succeed(HttpClient.HttpClient, httpClient), + Layer.succeed(Crypto.Crypto, crypto), + Layer.succeed(CommandSettings, cliSettings), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), ); + return Layer.mergeAll(deps, stackCatalogSetupLayer.pipe(Layer.provide(deps))); + }; const runtime = runtimeWith(output); const nextPort = (excluded?: number) => @@ -181,7 +195,10 @@ export const pgDeltaNextShadowLayer = Layer.effect( request.projectRef, request.toml.remoteOverrideKeys, ); - const image = yield* localInputs.resolvePostgresImage; + const image = + (yield* currentStackBackend).kind === "stack" + ? "stack-ephemeral" + : yield* localInputs.resolvePostgresImage; // One JWKS memo shared by every input built from this base: `provisionPlan`'s two // shadows must hash identical JWKS bytes or their snapshot keys can never match. return { @@ -271,6 +288,33 @@ export const pgDeltaNextShadowLayer = Layer.effect( } satisfies ProvisionedDeclarativeShadow; }).pipe(Effect.provide(runtimeWith(outputService)), Effect.mapError(nextShadowError)); + const stackAcquire = (input: NativeShadowInput, opts: ShadowCacheOpts) => + stackAcquireShadowDatabase(input.base, { + ...(opts.bypassCache === true ? { bypassCache: true } : {}), + port: input.base.shadowPort, + ...(opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }), + }); + + const stackProvisionMigrations = (input: NativeShadowInput, opts: ShadowCacheOpts) => + Effect.gen(function* () { + const handle = yield* stackAcquire(input, opts); + yield* stackMigrateShadow(handle, input.base); + return { + migrationsUrl: handle.url, + snapshotKey: handle.snapshotKey, + } satisfies ProvisionedMigrationsShadow; + }).pipe(Effect.provide(runtime), Effect.mapError(nextShadowError)); + + const stackProvisionDeclarative = (input: NativeShadowInput, opts: ShadowCacheOpts) => + Effect.gen(function* () { + const handle = yield* stackAcquire(input, opts); + return { + declarativeUrl: handle.url, + restoredFromPgDataSnapshot: handle.baselinePresent, + snapshotKey: handle.snapshotKey, + } satisfies ProvisionedDeclarativeShadow; + }).pipe(Effect.provide(runtime), Effect.mapError(nextShadowError)); + const cacheOpts = ( opts: PgDeltaNextShadowInput, webhooks: NonNullable, @@ -285,7 +329,10 @@ export const pgDeltaNextShadowLayer = Layer.effect( const port = yield* nextPort(); const built = yield* buildNativeBase(opts); const input = buildNativeInput(opts, built, port); - return yield* provisionMigrations(input, cacheOpts(opts, "config")); + const backend = yield* currentStackBackend; + return backend.kind === "stack" + ? yield* stackProvisionMigrations(input, cacheOpts(opts, "config")) + : yield* provisionMigrations(input, cacheOpts(opts, "config")); }).pipe(Effect.mapError(nextShadowError)), provisionPlan: (opts) => Effect.gen(function* () { @@ -294,6 +341,27 @@ export const pgDeltaNextShadowLayer = Layer.effect( const built = yield* buildNativeBase(opts); const migrationsInput = buildNativeInput(opts, built, migrationsPort); const declarativeInput = buildNativeInput(opts, built, declarativePort); + const backend = yield* currentStackBackend; + if (backend.kind === "stack") { + const migrations = yield* stackProvisionMigrations( + migrationsInput, + cacheOpts(opts, "config"), + ); + const declarative = yield* stackProvisionDeclarative( + declarativeInput, + cacheOpts(opts, "disabled"), + ); + return { + migrationsUrl: migrations.migrationsUrl, + declarativeUrl: declarative.declarativeUrl, + allowSameDatabaseIdentity: allowSameDatabaseIdentityForPlanShadows({ + declarativeRestoredFromPgDataSnapshot: declarative.restoredFromPgDataSnapshot, + sameSnapshotKey: + migrations.snapshotKey !== undefined && + migrations.snapshotKey === declarative.snapshotKey, + }), + } satisfies PgDeltaNextPlanShadows; + } const [migrationsPeek, declarativePeek] = yield* Effect.all([ peekShadowBaseline(migrationsInput.base, cacheOpts(opts, "config")), peekShadowBaseline(declarativeInput.base, cacheOpts(opts, "disabled")), diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts index 9f40101ac5..63a11c21fc 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts @@ -29,6 +29,7 @@ import { import { DockerRun } from "../../../command-internal/docker-run.service.ts"; import { dockerfileServiceImageRaw } from "../../../shared/services/dockerfile-images.ts"; import { SUGGEST_DOCKER_INSTALL } from "../../../command-internal/docker-suggest.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; import { DeclarativeShadowDbError } from "./pgdelta.errors.ts"; import { declarativeSeamLayer } from "./pgdelta.seam.layer.ts"; import { DeclarativeSeam } from "./pgdelta.seam.service.ts"; @@ -82,6 +83,7 @@ function setup( readonly failCreate?: boolean; readonly dbInspectFailsWith?: string; readonly dbInspectImage?: string; + readonly stackBackend?: boolean; } = {}, ) { const out = mockOutput(); @@ -134,6 +136,7 @@ function setup( Layer.succeed(DebugFlag, false), Layer.succeed(CliArgs, { args: [] }), seam, + ...(opts.stackBackend === true ? [stackBackendLayer("stack")] : []), ); return { layer, out, shadowSpawned: shadowSpawner.spawned }; @@ -226,4 +229,19 @@ describe("declarativeSeamLayer.ensureLocalPostgresImageCurrent", () => { rmSync(dir, { recursive: true, force: true }); }).pipe(Effect.provide(layer)); }); + + it.effect("skips docker container inspect when the stack backend is on", () => { + const dir = mkdtempSync(join(tmpdir(), "pgdelta-seam-")); + const { layer, shadowSpawned } = setup(dir, { + dbInspectImage: dockerfileServiceImageRaw("pg"), + stackBackend: true, + }); + return Effect.gen(function* () { + const seam = yield* DeclarativeSeam; + const exit = yield* seam.ensureLocalPostgresImageCurrent().pipe(Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + expect(shadowSpawned.some((s) => s.args.includes("inspect"))).toBe(false); + rmSync(dir, { recursive: true, force: true }); + }).pipe(Effect.provide(layer)); + }); }); diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts index 336fff70dd..64a1872f99 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts @@ -13,6 +13,9 @@ import { startLocalDatabase } from "../../../command-internal/db-bootstrap/start import { resolveLocalProjectId, localDbContainerId } from "../../../command-internal/docker-ids.ts"; import { DeclarativeShadowDbError } from "./pgdelta.errors.ts"; import { DeclarativeSeam } from "./pgdelta.seam.service.ts"; +import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; +import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts"; +import { stackEnsurePostgresOnlyStarted } from "../../../command-internal/stack-local-database.ts"; const shadowDockerCause = (stderr: string): { readonly docker: "daemon" } | Record => isDockerDaemonUnreachable(stderr) ? { docker: "daemon" } : {}; @@ -63,6 +66,7 @@ export const declarativeSeamLayer = Layer.effect( DeclarativeSeam, Effect.gen(function* () { const cliSettings = yield* CommandSettings; + const stackApi = yield* StackApi; const spawner = yield* ChildProcessSpawner; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -74,6 +78,22 @@ export const declarativeSeamLayer = Layer.effect( return DeclarativeSeam.of({ ensureLocalDatabaseStarted: () => Effect.gen(function* () { + const backend = yield* currentStackBackend; + if (backend.kind === "stack") { + return yield* stackEnsurePostgresOnlyStarted.pipe( + Effect.asVoid, + Effect.provideContext(context), + Effect.provideService(StackApi, stackApi), + Effect.mapError( + (cause) => + new DeclarativeShadowDbError({ + message: cause.message, + ...(cause.daemonDown === true ? { docker: "daemon" as const } : {}), + ...(cause.suggestion !== undefined ? { suggestion: cause.suggestion } : {}), + }), + ), + ); + } const running = yield* isLocalDbRunning( spawner, fs, @@ -112,134 +132,142 @@ export const declarativeSeamLayer = Layer.effect( ); }), ensureLocalPostgresImageCurrent: () => - Effect.scoped( - Effect.gen(function* () { - const toml = yield* readDbToml(fs, path, cliSettings.workdir).pipe( - Effect.mapError( - (error) => - new DeclarativeShadowDbError({ - message: `failed to read config for local Postgres image check: ${error.message}`, - }), - ), - ); - const { image } = yield* resolveDbImage( - fs, - path, - cliSettings.workdir, - toml.majorVersion, - Option.getOrUndefined(toml.orioledbVersion), - ); - const tomlProjectId = toml.projectId; - const projectId = resolveLocalProjectId( - Option.getOrUndefined(cliSettings.projectId), - Option.getOrUndefined(tomlProjectId), - cliSettings.workdir, - ); - const containerId = localDbContainerId(projectId); - const child = yield* spawnContainerCli(spawner, ["container", "inspect", containerId], { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - extendEnv: true, - }).pipe( - Effect.mapError( - () => - new DeclarativeShadowDbError({ - message: "failed to inspect local Postgres container.", - docker: "daemon", - }), - ), - ); - const stdoutChunks: Array = []; - const stderrChunks: Array = []; - yield* Stream.runForEach(child.stdout, (chunk) => - Effect.sync(() => { - stdoutChunks.push(chunk); - }), - ).pipe( - Effect.mapError( - () => - new DeclarativeShadowDbError({ - message: "failed to inspect local Postgres container.", - docker: "daemon", - }), - ), - ); - yield* Stream.runForEach(child.stderr, (chunk) => - Effect.sync(() => { - stderrChunks.push(chunk); - }), - ).pipe( - Effect.mapError( - () => - new DeclarativeShadowDbError({ - message: "failed to inspect local Postgres container.", - docker: "daemon", - }), - ), - ); - const inspectExit = yield* child.exitCode.pipe( - Effect.map(Number), - Effect.mapError( - () => + Effect.gen(function* () { + const backend = yield* currentStackBackend; + if (backend.kind === "stack") return; + return yield* Effect.scoped( + Effect.gen(function* () { + const toml = yield* readDbToml(fs, path, cliSettings.workdir).pipe( + Effect.mapError( + (error) => + new DeclarativeShadowDbError({ + message: `failed to read config for local Postgres image check: ${error.message}`, + }), + ), + ); + const { image } = yield* resolveDbImage( + fs, + path, + cliSettings.workdir, + toml.majorVersion, + Option.getOrUndefined(toml.orioledbVersion), + ); + const tomlProjectId = toml.projectId; + const projectId = resolveLocalProjectId( + Option.getOrUndefined(cliSettings.projectId), + Option.getOrUndefined(tomlProjectId), + cliSettings.workdir, + ); + const containerId = localDbContainerId(projectId); + const child = yield* spawnContainerCli( + spawner, + ["container", "inspect", containerId], + { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + extendEnv: true, + }, + ).pipe( + Effect.mapError( + () => + new DeclarativeShadowDbError({ + message: "failed to inspect local Postgres container.", + docker: "daemon", + }), + ), + ); + const stdoutChunks: Array = []; + const stderrChunks: Array = []; + yield* Stream.runForEach(child.stdout, (chunk) => + Effect.sync(() => { + stdoutChunks.push(chunk); + }), + ).pipe( + Effect.mapError( + () => + new DeclarativeShadowDbError({ + message: "failed to inspect local Postgres container.", + docker: "daemon", + }), + ), + ); + yield* Stream.runForEach(child.stderr, (chunk) => + Effect.sync(() => { + stderrChunks.push(chunk); + }), + ).pipe( + Effect.mapError( + () => + new DeclarativeShadowDbError({ + message: "failed to inspect local Postgres container.", + docker: "daemon", + }), + ), + ); + const inspectExit = yield* child.exitCode.pipe( + Effect.map(Number), + Effect.mapError( + () => + new DeclarativeShadowDbError({ + message: "failed to inspect local Postgres container.", + docker: "daemon", + }), + ), + ); + const decodeChunks = (chunks: ReadonlyArray): string => { + const total = chunks.reduce((size, chunk) => size + chunk.length, 0); + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return new TextDecoder().decode(bytes).trim(); + }; + const stderr = decodeChunks(stderrChunks); + const stdout = decodeChunks(stdoutChunks); + if (inspectExit !== 0) { + if (isMissingContainerInspectError(stderr)) return; + return yield* Effect.fail( new DeclarativeShadowDbError({ - message: "failed to inspect local Postgres container.", - docker: "daemon", + message: + stderr.length > 0 + ? `failed to inspect local Postgres container: ${stderr}` + : "failed to inspect local Postgres container.", + ...shadowDockerCause(stderr), }), - ), - ); - const decodeChunks = (chunks: ReadonlyArray): string => { - const total = chunks.reduce((size, chunk) => size + chunk.length, 0); - const bytes = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.length; + ); } - return new TextDecoder().decode(bytes).trim(); - }; - const stderr = decodeChunks(stderrChunks); - const stdout = decodeChunks(stdoutChunks); - if (inspectExit !== 0) { - if (isMissingContainerInspectError(stderr)) return; + const actual = resolveContainerInspectImageName(stdout); + const expected = getRegistryImageUrl(image).trim(); + const actualTag = dockerImageTag(actual); + const expectedTag = dockerImageTag(expected); + if (actual.length === 0 || actualTag.length === 0 || expectedTag.length === 0) { + return; + } + // Slim refs never go through a registry mirror, so a family mismatch + // (e.g. a docker.io container satisfying a ghcr.io/supabase/cli + // expectation) is stale even when the tags happen to match. + const familyMismatch = isSlimImageRef(expected) !== isSlimImageRef(actual); + if (!familyMismatch && actualTag === expectedTag) { + return; + } + const remediation = + familyMismatch && actualTag === expectedTag + ? "The tags match but the image family does not (slim vs docker.io). Run supabase stop, then supabase start with the same SUPABASE_USE_SLIM_IMAGES setting before syncing declarative schemas." + : "Run supabase stop --all --no-backup, then supabase start before syncing declarative schemas."; return yield* Effect.fail( new DeclarativeShadowDbError({ - message: - stderr.length > 0 - ? `failed to inspect local Postgres container: ${stderr}` - : "failed to inspect local Postgres container.", - ...shadowDockerCause(stderr), + message: `local Postgres container image is stale: running ${actual} but expected ${expected}. ${remediation}`, }), ); - } - const actual = resolveContainerInspectImageName(stdout); - const expected = getRegistryImageUrl(image).trim(); - const actualTag = dockerImageTag(actual); - const expectedTag = dockerImageTag(expected); - if (actual.length === 0 || actualTag.length === 0 || expectedTag.length === 0) { - return; - } - // Slim refs never go through a registry mirror, so a family mismatch - // (e.g. a docker.io container satisfying a ghcr.io/supabase/cli - // expectation) is stale even when the tags happen to match. - const familyMismatch = isSlimImageRef(expected) !== isSlimImageRef(actual); - if (!familyMismatch && actualTag === expectedTag) { - return; - } - const remediation = - familyMismatch && actualTag === expectedTag - ? "The tags match but the image family does not (slim vs docker.io). Run supabase stop, then supabase start with the same SUPABASE_USE_SLIM_IMAGES setting before syncing declarative schemas." - : "Run supabase stop --all --no-backup, then supabase start before syncing declarative schemas."; - return yield* Effect.fail( - new DeclarativeShadowDbError({ - message: `local Postgres container image is stale: running ${actual} but expected ${expected}. ${remediation}`, - }), - ); - }), - ), + }), + ); + }), }); }), -); +).pipe(Layer.provide(stackApiLayer)); type StartLocalDatabaseDeps = ReturnType extends Effect.Effect diff --git a/apps/cli/src/commands/db/start/start.errors.ts b/apps/cli/src/commands/db/start/start.errors.ts new file mode 100644 index 0000000000..1410f17c24 --- /dev/null +++ b/apps/cli/src/commands/db/start/start.errors.ts @@ -0,0 +1,18 @@ +import { Data } from "effect"; +import { + actionability, + type CliErrorActionabilityDeclaration, + ErrorActionabilityId, +} from "../../../shared/telemetry/error-actionability.ts"; + +/** `--from-backup` restore is Compose-only; stack `db start` has no restore path. */ +export class DbStartFromBackupUnsupportedError extends Data.TaggedError( + "DbStartFromBackupUnsupportedError", +)<{ + readonly message: string; + readonly suggestion?: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.provideFlags; + } +} diff --git a/apps/cli/src/commands/db/start/start.handler.ts b/apps/cli/src/commands/db/start/start.handler.ts index 3e6f9b9b35..c3185eedbb 100644 --- a/apps/cli/src/commands/db/start/start.handler.ts +++ b/apps/cli/src/commands/db/start/start.handler.ts @@ -3,6 +3,9 @@ import { Effect, Option } from "effect"; import { Output } from "../../../shared/output/output.service.ts"; import { TelemetryState } from "../../../telemetry/telemetry-state.service.ts"; import { startLocalDatabase } from "../../../command-internal/db-bootstrap/start-local-database.ts"; +import { stackEnsurePostgresOnlyStarted } from "../../../command-internal/stack-local-database.ts"; +import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; +import { DbStartFromBackupUnsupportedError } from "./start.errors.ts"; import type { DbStartFlags } from "./start.command.ts"; /** @@ -11,13 +14,45 @@ import type { DbStartFlags } from "./start.command.ts"; * `ensureLocalDatabaseStarted`. This handler only adds the output-format-aware terminal * message and telemetry flush. Unlike `supabase start`, it has no status table, no * `cli_stack_started` event, and no `--exclude`/`--ignore-health-check` flags. + * + * When `[experimental].stack` is on, this command starts a postgres-only project stack + * instead of Compose. `--from-backup` is refused on the stack path. */ export const dbStart = Effect.fn("db.start")(function* (flags: DbStartFlags) { const output = yield* Output; const telemetryState = yield* TelemetryState; const body = Effect.gen(function* () { - const result = yield* startLocalDatabase(Option.getOrUndefined(flags.fromBackup)); + const backend = yield* currentStackBackend; + const fromBackup = Option.getOrUndefined(flags.fromBackup); + if (backend.kind === "stack") { + if (fromBackup !== undefined && fromBackup.length > 0) { + return yield* Effect.fail( + new DbStartFromBackupUnsupportedError({ + message: "db start --from-backup is not supported when the stack backend is enabled.", + suggestion: + "Omit --from-backup, or disable [experimental].stack to restore a Compose backup.", + }), + ); + } + const result = yield* stackEnsurePostgresOnlyStarted; + if (result === "already-running") { + if (output.format === "text") { + yield* output.raw("Postgres database is already running.\n", "stderr"); + } else { + yield* output.success("Postgres database is already running.", { + status: "already-running", + }); + } + return; + } + if (output.format !== "text") { + yield* output.success("Started local database.", { status: "started" }); + } + return; + } + + const result = yield* startLocalDatabase(fromBackup); if (result.status === "already-running") { if (output.format === "text") { diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 5261bf8e5c..c2e16c982b 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -3,7 +3,7 @@ import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { afterEach, beforeEach, describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option, PlatformError, Sink, Stream } from "effect"; +import { Cause, Effect, Exit, Layer, Option, PlatformError, Sink, Stream, Redacted } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; @@ -34,6 +34,13 @@ import { DbConnection, type DbSession } from "../../../command-internal/db-conne import { dockerRunLayer } from "../../../command-internal/docker-run.layer.ts"; import { dbStart } from "./start.handler.ts"; import type { DbStartFlags } from "./start.command.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; +import { StackApi } from "../../../command-internal/stack-api.ts"; +import { + noopStackCatalogSetupLayer, + recordingStackCatalogSetup, +} from "../../../command-internal/stack-catalog-setup.ts"; +import { CAPABILITY_NAMES, StackIdSchema, type EffectStack } from "@supabase/stack/effect"; const DEFAULT_FLAGS: DbStartFlags = { fromBackup: Option.none() }; const PG_NET_CREATE_FINGERPRINT = "create extension if not exists pg_net schema extensions"; @@ -275,9 +282,15 @@ interface SetupOpts { readonly connectFailures?: number; /** Whether the mocked connect failures are dial-level (`retryable`). Defaults to `true`. */ readonly connectFailuresRetryable?: boolean; + /** Record catalog apply targets instead of the default noop. */ + readonly recordCatalog?: boolean; } function setup(opts: SetupOpts = {}) { + const catalog = + opts.recordCatalog === true + ? recordingStackCatalogSetup((input) => input.target.kind) + : undefined; const workdir = opts.workdir ?? tempRoot.current; if (opts.skipConfig !== true) { writeConfig(workdir, opts.configContents ?? 'project_id = "test"\n'); @@ -336,6 +349,7 @@ function setup(opts: SetupOpts = {}) { Layer.succeed(CliArgs, { args: ["db", "start"] }), Layer.succeed(ExperimentalFlag, opts.experimental ?? false), Layer.succeed(DebugFlag, opts.debug ?? false), + catalog?.layer ?? noopStackCatalogSetupLayer, ); return { layer, @@ -343,6 +357,7 @@ function setup(opts: SetupOpts = {}) { telemetry, child, dbSession, + catalogApplied: catalog?.applied ?? [], get connectAttempts() { return connectAttempts; }, @@ -1518,3 +1533,168 @@ describe("db start", () => { }); }); }); + +describe("db start stack backend", () => { + const STACK_ID = StackIdSchema.make("b".repeat(64)); + const unused = () => Effect.die("unused"); + const unusedEffect = Effect.die("unused"); + + function mockStackApi(opts: { + readonly existing?: boolean; + readonly unconfigured?: boolean; + readonly databaseReady?: boolean; + }) { + const startConfigs: Array = []; + const stack: EffectStack = { + id: STACK_ID, + status: Effect.succeed({ + id: STACK_ID, + lifecycle: opts.databaseReady === true ? "running" : "stopped", + desiredLifecycle: opts.databaseReady === true ? "running" : "stopped", + runtime: { kind: "native" }, + endpoints: {}, + versions: {}, + capabilities: CAPABILITY_NAMES.map((name) => ({ + name, + activation: name === "database" ? "eager" : "lazy", + state: name === "database" && opts.databaseReady === true ? "ready" : "stopped", + })), + artifacts: [], + }), + credentials: Effect.succeed({ + database: { + url: Redacted.make("postgresql://postgres:secret@127.0.0.1:54329/postgres"), + password: Redacted.make("secret"), + }, + api: { + publishableKey: "anon", + secretKey: Redacted.make("service"), + anonJwt: "anon", + serviceRoleJwt: Redacted.make("service"), + }, + }), + prepare: unused, + start: (startOpts) => + Effect.sync(() => { + startConfigs.push(startOpts?.config); + return { + id: STACK_ID, + lifecycle: "running" as const, + desiredLifecycle: "running" as const, + runtime: { kind: "native" as const }, + endpoints: {}, + versions: {}, + capabilities: CAPABILITY_NAMES.map((name) => ({ + name, + activation: name === "database" ? ("eager" as const) : ("lazy" as const), + state: name === "database" ? ("ready" as const) : ("dormant" as const), + })), + artifacts: [], + }; + }), + stop: unusedEffect, + destroy: unusedEffect, + resetDatabase: unusedEffect, + logs: unused, + followLogs: () => Stream.empty, + }; + const api = Layer.succeed(StackApi, { + createStack: () => Effect.succeed(stack), + findStack: () => + Effect.succeed( + opts.existing === true + ? Option.some({ + id: STACK_ID, + projectRoot: tempRoot.current, + name: "default", + branchContext: "main", + runtime: { kind: "native" as const }, + desiredLifecycle: + opts.unconfigured === true ? ("unconfigured" as const) : ("stopped" as const), + }) + : Option.none(), + ), + discoverStacks: unused, + openStack: () => Effect.succeed(stack), + inspectStack: unused, + }); + return { api, startConfigs }; + } + + it.live("starts a postgres-only stack when none exists", () => { + const { layer, catalogApplied } = setup({ recordCatalog: true }); + const stack = mockStackApi({}); + return Effect.gen(function* () { + yield* dbStart(DEFAULT_FLAGS).pipe( + Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + ); + expect(stack.startConfigs).toHaveLength(1); + expect(stack.startConfigs[0]).toMatchObject({ + capabilities: { + rest: { enabled: false }, + }, + }); + expect(catalogApplied).toEqual(["live"]); + }); + }); + + it.live("does not persist exclusions when a stack already exists", () => { + const { layer, catalogApplied } = setup({ recordCatalog: true }); + const stack = mockStackApi({ existing: true }); + return Effect.gen(function* () { + yield* dbStart(DEFAULT_FLAGS).pipe( + Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + ); + expect(stack.startConfigs).toEqual([undefined]); + expect(catalogApplied).toEqual(["live"]); + }); + }); + + it.live("applies the postgres-only overlay when an unconfigured identity already exists", () => { + const { layer, catalogApplied } = setup({ recordCatalog: true }); + const stack = mockStackApi({ existing: true, unconfigured: true }); + return Effect.gen(function* () { + yield* dbStart(DEFAULT_FLAGS).pipe( + Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + ); + expect(stack.startConfigs).toHaveLength(1); + expect(stack.startConfigs[0]).toMatchObject({ + capabilities: { + rest: { enabled: false }, + }, + }); + expect(catalogApplied).toEqual(["live"]); + }); + }); + + it.live("reports an already-running stack database without starting", () => { + const { layer, out, catalogApplied } = setup({ recordCatalog: true }); + const stack = mockStackApi({ existing: true, databaseReady: true }); + return Effect.gen(function* () { + yield* dbStart(DEFAULT_FLAGS).pipe( + Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + ); + expect(out.stderrText).toContain("Postgres database is already running."); + expect(stack.startConfigs).toEqual([]); + expect(catalogApplied).toEqual(["live"]); + }); + }); + + it.live("refuses --from-backup", () => { + const { layer } = setup(); + const stack = mockStackApi({}); + return Effect.gen(function* () { + const exit = yield* dbStart(flags("backup.sql")).pipe( + Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + Effect.exit, + ); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(JSON.stringify(exit.cause)).toContain( + "db start --from-backup is not supported when the stack backend is enabled.", + ); + } + expect(stack.startConfigs).toEqual([]); + }); + }); +}); diff --git a/apps/cli/src/commands/db/start/start.layers.ts b/apps/cli/src/commands/db/start/start.layers.ts index 7725681631..41c3eb2661 100644 --- a/apps/cli/src/commands/db/start/start.layers.ts +++ b/apps/cli/src/commands/db/start/start.layers.ts @@ -8,6 +8,8 @@ import { dbConnectionLayer } from "../../../command-internal/db-connection.layer import { debugLoggerLayer } from "../../../command-internal/debug-logger.layer.ts"; import { dockerRunLayer } from "../../../command-internal/docker-run.layer.ts"; import { telemetryStateLayer } from "../../../telemetry/telemetry-state.layer.ts"; +import { stackApiLayer } from "../../../command-internal/stack-api.ts"; +import { stackCatalogSetupLayer } from "../../../command-internal/stack-catalog-setup.ts"; /** * Runtime layer for `supabase db start`, matching `supabase start`'s own composition. @@ -28,4 +30,6 @@ export const dbStartRuntimeLayer = Layer.mergeAll( dockerRunLayer, dbConnectionLayer, httpClient, + stackApiLayer, + stackCatalogSetupLayer, ); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts index 93f15c7015..4ba59a508c 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts @@ -67,6 +67,7 @@ function setup(options: { : options.destroyFailure ? Effect.fail(new StackDestructionError({ message: "destroy failed" })) : Effect.sync(() => void state.destroyed++), + resetDatabase: Effect.die("unused"), logs: () => Effect.die("unused"), followLogs: () => Stream.empty, }; diff --git a/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts index 18f20afe78..46ed9b84a8 100644 --- a/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Effect, Exit, Option } from "effect"; import { respondToComplete } from "../../../cli/complete.ts"; import { rootCommandForFeatures } from "../../../cli/root.ts"; -import { StackRoutingError, resolveStackBackend } from "./stack-backend.ts"; +import { StackRoutingError, resolveStackBackend } from "../../../command-internal/stack-backend.ts"; const resolve = (input: Parameters[0]) => resolveStackBackend(input).pipe(Effect.provide(BunServices.layer)); @@ -51,6 +51,10 @@ stack = true expect(yield* resolve({ args: ["start"], cwd: join(root, "nested"), env: {} })).toBe("stack"); expect(yield* resolve({ args: ["stop"], cwd: root, env: {} })).toBe("stack"); expect(yield* resolve({ args: ["status"], cwd: root, env: {} })).toBe("legacy"); + expect(yield* resolve({ args: ["db", "diff"], cwd: root, env: {} })).toBe("stack"); + expect(yield* resolve({ args: ["db", "test"], cwd: root, env: {} })).toBe("stack"); + expect(yield* resolve({ args: ["test", "db"], cwd: root, env: {} })).toBe("stack"); + expect(yield* resolve({ args: ["migration", "squash"], cwd: root, env: {} })).toBe("stack"); }).pipe(Effect.ensuring(Effect.sync(() => rmSync(root, { recursive: true, force: true })))); }); diff --git a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts index 4be732261e..3b58fd89d5 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts @@ -7,7 +7,7 @@ import { Cause, Effect, Exit, Option, Redacted } from "effect"; import { compileStack } from "../../../../../../packages/stack/src/model/Compiler.ts"; import { withEnvVar } from "../../../../tests/helpers/command-mocks.ts"; -import { StackConfigError, loadStackConfig } from "./stack-config.ts"; +import { StackConfigError, loadStackConfig } from "../../../command-internal/stack-config.ts"; import { createStackConfigProject } from "../../../../tests/helpers/stack-config.ts"; function withEnvironment( diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index 607a0a26e6..aff5419dae 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Effect, Exit, Option, Path, Redacted } from "effect"; import { renderCliConfigTemplate } from "../../../shared/init/project-init.templates.ts"; -import { StackConfigError, loadStackConfig } from "./stack-config.ts"; +import { StackConfigError, loadStackConfig } from "../../../command-internal/stack-config.ts"; import { createStackConfigProject, stackConfigTempRoot, diff --git a/apps/cli/src/commands/experimental/stack/stack.command.ts b/apps/cli/src/commands/experimental/stack/stack.command.ts index ddea323de6..0894f9f328 100644 --- a/apps/cli/src/commands/experimental/stack/stack.command.ts +++ b/apps/cli/src/commands/experimental/stack/stack.command.ts @@ -4,6 +4,8 @@ import { commandRuntimeLayer } from "../../../shared/runtime/command-runtime.lay import { commandSettingsLayer } from "../../../config/command-settings.layer.ts"; import { debugLoggerLayer } from "../../../command-internal/debug-logger.layer.ts"; import { telemetryStateLayer } from "../../../telemetry/telemetry-state.layer.ts"; +import { dbConnectionLayer } from "../../../command-internal/db-connection.layer.ts"; +import { stackCatalogSetupLayer } from "../../../command-internal/stack-catalog-setup.ts"; import { stackStartCommand as stackStartCommandBase } from "./start/start.command.ts"; import { stackStopCommand as stackStopCommandBase } from "./stop/stop.command.ts"; import { stackDestroyCommand as stackDestroyCommandBase } from "./destroy/destroy.command.ts"; @@ -12,6 +14,8 @@ import { stackApiLayer, stackTargetResolverLayer } from "./stack.shared.ts"; export const stackRuntimeLayer = Layer.mergeAll( stackTargetResolverLayer, stackApiLayer, + dbConnectionLayer, + stackCatalogSetupLayer, commandSettingsLayer.pipe(Layer.provide(debugLoggerLayer)), telemetryStateLayer, ); diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.ts b/apps/cli/src/commands/experimental/stack/stack.shared.ts index d098356e40..f3b2afeb75 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.ts @@ -1,22 +1,14 @@ -import { Context, Data, Effect, FileSystem, Layer, Option, Path, Crypto } from "effect"; -import { - createStack, - discoverStacks, - findStack, - inspectStack, - isStackId, - openStack, - type StackRuntimePreference, - type StackDiscoveryResult, -} from "@supabase/stack/effect"; +import { Context, Data, Effect, Layer, Option } from "effect"; +import { isStackId, StackNotFoundError, type StackRuntimePreference } from "@supabase/stack/effect"; import type { StackId } from "@supabase/stack"; -import { StackNotFoundError } from "@supabase/stack/effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; import { actionability, type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../../../shared/telemetry/error-actionability.ts"; +import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts"; + +export { StackApi, stackApiLayer }; /** The target selected by the CLI adapter for one stack command. */ interface StackTarget { @@ -55,39 +47,6 @@ export class StackTargetResolver extends Context.Service< StackTargetResolverShape >()("supabase/experimental-stack/TargetResolver") {} -export class StackApi extends Context.Service< - StackApi, - { - readonly findStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly createStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly openStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly inspectStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly discoverStacks: ( - ...args: Parameters - ) => Effect.Effect>>; - } ->()("supabase/experimental-stack/StackApi") {} - export const validateStackTarget = (input: { readonly stack?: string; readonly stackId?: string; @@ -125,33 +84,6 @@ export const rejectStackOutput = ( ) : Effect.void; -export const stackApiLayer = Layer.effect( - StackApi, - Effect.gen(function* () { - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const childProcess = yield* ChildProcessSpawner.ChildProcessSpawner; - const provideServices = (effect: Effect.Effect) => - effect.pipe( - Effect.provideService(FileSystem.FileSystem, fileSystem), - Effect.provideService(Path.Path, path), - Effect.provideService(Crypto.Crypto, crypto), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcess), - ); - return { - findStack: (...args: Parameters) => provideServices(findStack(...args)), - createStack: (...args: Parameters) => - provideServices(createStack(...args)), - openStack: (...args: Parameters) => provideServices(openStack(...args)), - inspectStack: (...args: Parameters) => - provideServices(inspectStack(...args)), - discoverStacks: (...args: Parameters) => - provideServices(discoverStacks(...args)), - }; - }), -); - /** Runtime configuration for the first stack command. Later commands reuse this layer. */ export const stackTargetResolverLayer = Layer.succeed(StackTargetResolver, { resolve: (input) => diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 8bfc8ef9d0..355987ab9c 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -59,7 +59,8 @@ activated before the command returns. `storage`, `functions`, `studio`, `mail`, `analytics`, and `pooler`) and disables those services in the effective start configuration. The database cannot be excluded. Exclusions are applied in memory and persisted with the stack state; the project configuration file is unchanged. A capability -and its dependents are disabled together, so excluding `rest` or `analytics` also disables `studio`. +and its dependents are disabled together, so excluding `rest` also disables `studio`. Excluding +`analytics` does not. Listeners are derived by the runtime from enabled capability routes; route-less listeners are therefore omitted. Eager activation never re-enables an excluded capability. diff --git a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts index 8934240254..b620526c8b 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts @@ -57,12 +57,16 @@ async function inspectStackState(home: string, stackId: string) { const inspection = await inspectStack(id); const stack = await openStack(id); const status = await stack.status(); + const credentials = + status.lifecycle === "running" ? await stack.credentials() : undefined; console.log(JSON.stringify({ owner: inspection.owner, projectRoot: inspection.descriptor.projectRoot, runtime: status.runtime, lifecycle: status.lifecycle, database: status.capabilities.find(({ name }) => name === "database")?.state, + databaseUrl: credentials?.database.url, + hasApi: credentials?.api !== undefined, })); `; const result = await execFile("bun", ["--bun", "-e", script, stackId], { @@ -83,6 +87,8 @@ async function inspectStackState(home: string, stackId: string) { readonly runtime: { readonly kind: string }; readonly lifecycle: string; readonly database: string | undefined; + readonly databaseUrl: string | undefined; + readonly hasApi: boolean; }; } @@ -169,6 +175,10 @@ describe("stack start (compiled e2e)", () => { expect(running.runtime).toEqual({ kind: "native" }); expect(running.lifecycle).toBe("running"); expect(running.database).toBe("ready"); + expect(running.hasApi).toBe(false); + expect(running.databaseUrl).toMatch( + /^postgresql:\/\/postgres:.+@127\.0\.0\.1:\d+\/postgres$/, + ); const databasePath = path.join(homeDir.dir, "managed", "stacks", idText, "data", "database"); await access(path.join(databasePath, "PG_VERSION")); diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 316c25048b..9ca8ab5d9a 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -1,4 +1,4 @@ -import { Effect, Match, Option } from "effect"; +import { Effect, FileSystem, Match, Option, Path } from "effect"; import { excludeStackCapabilities, isStackError, @@ -9,6 +9,8 @@ import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; +import { readDbToml } from "../../../../command-internal/db-config.toml-read.ts"; +import { StackCatalogSetup } from "../../../../command-internal/stack-catalog-setup.ts"; import { StackApi, StackTargetError, @@ -16,7 +18,7 @@ import { rejectStackOutput, validateStackTarget, } from "../stack.shared.ts"; -import { loadStackConfig } from "../stack-config.ts"; +import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import type { StackStartFlags } from "./start.command.ts"; import { StackCommandStartError } from "./start.errors.ts"; import { STACK_START_EXCLUDABLE_CAPABILITIES } from "./start.options.ts"; @@ -180,9 +182,55 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags const starting = yield* output.task("Starting local Supabase stack..."); const status = yield* stack.start({ config: startConfig }).pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.tap(() => starting.succeed("Stack is ready.")), Effect.mapError(stackStartError), ); + const catalog = yield* Effect.serviceOption(StackCatalogSetup); + if (Option.isNone(catalog)) + return yield* new StackCommandStartError({ + reason: "unknown", + message: "stack catalog setup is unavailable", + }); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const toml = yield* readDbToml(fs, path, target.projectRoot).pipe( + Effect.mapError( + (error) => + new StackCommandStartError({ + reason: "invalid-config", + message: error.message, + cause: error, + }), + ), + ); + yield* catalog.value + .apply({ + target: { + kind: "live", + stack, + projectRoot: target.projectRoot, + config, + }, + overlay: { + webhooks: "config", + webhooksEnabled: toml.webhooksEnabled, + apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, + vault: toml.vault, + workdir: target.projectRoot, + }, + }) + .pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError((error) => + isStackError(error.cause) + ? stackStartError(error.cause) + : new StackCommandStartError({ + reason: "unknown", + message: error.message, + cause: error, + }), + ), + ); + yield* starting.succeed("Stack is ready."); if (output.format === "text") { yield* output.raw(renderStatus(status)); } else { diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index c9eb301203..b91bdb8aea 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -44,6 +44,10 @@ import { actionability, ErrorActionabilityId, } from "../../../../shared/telemetry/error-actionability.ts"; +import { + noopStackCatalogSetupLayer, + recordingStackCatalogSetup, +} from "../../../../command-internal/stack-catalog-setup.ts"; const project = (): string => { const root = mkdtempSync(join(tmpdir(), "supabase-experimental-stack-start-")); @@ -101,6 +105,7 @@ function fakeStack( start, stop: Effect.void, destroy: Effect.die("destroy not used in start test"), + resetDatabase: Effect.die("resetDatabase not used in start test"), logs: () => Effect.die("logs not used in start test"), followLogs: () => Stream.empty, } satisfies EffectStack; @@ -162,13 +167,17 @@ function handlerLayer(opts: { targetLayer, apiLayer, BunServices.layer, + noopStackCatalogSetupLayer, ), }; } describe("stack start targeting", () => { - for (const exclusion of ["rest", "analytics"] as const) { - it.live(`compiles ${exclusion} exclusion and dependent Studio`, () => { + for (const { exclusion, studioEnabled } of [ + { exclusion: "rest" as const, studioEnabled: false }, + { exclusion: "analytics" as const, studioEnabled: true }, + ]) { + it.live(`compiles ${exclusion} exclusion with Studio ${studioEnabled ? "on" : "off"}`, () => { const root = project(); const configBefore = readFileSync(join(root, "supabase", "config.toml"), "utf8"); const stack = fakeStack("c".repeat(64), (input) => @@ -187,7 +196,7 @@ describe("stack start targeting", () => { Effect.provide(BunServices.layer), ); expect(compiled.definition.capabilities[exclusion].enabled).toBe(false); - expect(compiled.definition.capabilities.studio.enabled).toBe(false); + expect(compiled.definition.capabilities.studio.enabled).toBe(studioEnabled); expect(compiled.definition.capabilities.auth.enabled).toBe(true); return status("c".repeat(64)); }), @@ -237,6 +246,23 @@ describe("stack start targeting", () => { ); }); + it.live("applies catalog setup from pre-exclude config after start returns", () => { + const root = project(); + const catalog = recordingStackCatalogSetup((input) => ({ + kind: input.target.kind, + authEnabled: input.target.config.capabilities?.auth?.enabled, + })); + const stack = fakeStack("f".repeat(64), () => Effect.succeed(status("f".repeat(64)))); + const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); + return Effect.gen(function* () { + yield* stackStart(flags({ exclude: ["auth"] })); + expect(catalog.applied).toEqual([{ kind: "live", authEnabled: undefined }]); + }).pipe( + Effect.provide(Layer.mergeAll(setup.layer, catalog.layer)), + Effect.ensuring(Effect.sync(() => rmSync(root, { recursive: true, force: true }))), + ); + }); + it.live( "leaves listener configuration to the compiled runtime when capabilities are excluded", () => { diff --git a/apps/cli/src/commands/experimental/stack/start/start.options.ts b/apps/cli/src/commands/experimental/stack/start/start.options.ts index 28c5e7e730..55147662bb 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.options.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.options.ts @@ -1,6 +1 @@ -import { CAPABILITY_NAMES } from "@supabase/stack/effect"; - -/** Optional capabilities accepted by `stack start --exclude`. */ -export const STACK_START_EXCLUDABLE_CAPABILITIES = CAPABILITY_NAMES.filter( - (name) => name !== "database", -); +export { STACK_START_EXCLUDABLE_CAPABILITIES } from "../../../../command-internal/stack-local-database.ts"; diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts index f38cb3c40d..139da02e70 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts @@ -96,6 +96,7 @@ function setup(opts: { destroy: Effect.sync(() => { state.destroyCalled = true; }), + resetDatabase: Effect.die("unused"), logs: () => Effect.die("unused"), followLogs: () => Stream.empty, } satisfies EffectStack; diff --git a/apps/cli/src/commands/migration/migration.layers.ts b/apps/cli/src/commands/migration/migration.layers.ts index e9366c12cc..5fd1e1f9bf 100644 --- a/apps/cli/src/commands/migration/migration.layers.ts +++ b/apps/cli/src/commands/migration/migration.layers.ts @@ -11,6 +11,9 @@ import { dockerRunLayer } from "../../command-internal/docker-run.layer.ts"; import { identityStitchLayer } from "../../command-internal/identity-stitch.ts"; import { linkedDbResolverRuntimeLayer } from "../../command-internal/management-api-runtime.layer.ts"; import { telemetryStateLayer } from "../../telemetry/telemetry-state.layer.ts"; +import { stackApiLayer } from "../../command-internal/stack-api.ts"; +import { ephemeralPostgresLayer } from "../../command-internal/stack-shadow.ts"; +import { stackCatalogSetupLayer } from "../../command-internal/stack-catalog-setup.ts"; const cliSettings = commandSettingsLayer.pipe(Layer.provide(debugLoggerLayer)); @@ -64,4 +67,7 @@ export const migrationSquashRuntimeLayer = Layer.mergeAll( dockerRunLayer, httpClient, debugLoggerLayer, + stackApiLayer, + ephemeralPostgresLayer, + stackCatalogSetupLayer, ); diff --git a/apps/cli/src/commands/migration/squash/squash.dump.ts b/apps/cli/src/commands/migration/squash/squash.dump.ts index 8b7c897a15..2809158709 100644 --- a/apps/cli/src/commands/migration/squash/squash.dump.ts +++ b/apps/cli/src/commands/migration/squash/squash.dump.ts @@ -3,7 +3,11 @@ import { Effect } from "effect"; import type { PgConnInput } from "../../../command-internal/db-connection.service.ts"; import { buildSchemaDumpEnv, type DumpOptions } from "../../../command-internal/pg-dump.env.ts"; import { dumpSchemaScript } from "../../../command-internal/pg-dump.scripts.ts"; -import { streamPgDump } from "../../../command-internal/pg-dump.run.ts"; +import { + pgDumpClientExitMessage, + streamPgDumpWithClient, + type PgDumpClient, +} from "../../../command-internal/pg-dump.run.ts"; import { MigrationSquashDumpError } from "./squash.errors.ts"; /** @@ -25,6 +29,8 @@ export interface SquashDumpParams { readonly onStdout: (chunk: Uint8Array) => Effect.Effect; /** Loaded project `supabase/.env` map — forwarded to {@link streamPgDump}'s own `SUPABASE_NETWORK_ID` fallback. */ readonly projectEnvValues?: Readonly>; + /** Native-engine shadows dump with PATH `pg_dump`; container shadows keep the tool container. */ + readonly client?: PgDumpClient; } /** @@ -40,34 +46,24 @@ export const squashDumpSchema = Effect.fnUntraced(function* (params: SquashDu excludeTable: [], columnInsert: false, }; - const result = yield* streamPgDump({ + const client = params.client ?? { kind: "container" as const }; + const result = yield* streamPgDumpWithClient({ image: params.image, script: dumpSchemaScript, env: buildSchemaDumpEnv(params.conn, opt), onStdout: params.onStdout, projectEnvValues: params.projectEnvValues, + client, }); if (result.exitCode !== 0) { return yield* Effect.fail( new MigrationSquashDumpError({ - message: `error running container: exit ${result.exitCode}`, + message: pgDumpClientExitMessage(client, result.exitCode), }), ); } }); -/** Concatenates stdout chunks into one buffer. */ -const concatChunks = (chunks: ReadonlyArray): Uint8Array => { - const total = chunks.reduce((size, chunk) => size + chunk.length, 0); - const bytes = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.length; - } - return bytes; -}; - /** * Buffered convenience over {@link squashDumpSchema} for the before/after * diff dumps — an `auth`/`storage` schema-only dump is tens of KB, not @@ -80,6 +76,7 @@ export const squashDumpSchemaToString = Effect.fnUntraced(function* (params: { readonly conn: PgConnInput; readonly schema: ReadonlyArray; readonly projectEnvValues?: Readonly>; + readonly client?: PgDumpClient; }) { const chunks: Array = []; yield* squashDumpSchema({ @@ -88,6 +85,7 @@ export const squashDumpSchemaToString = Effect.fnUntraced(function* (params: { schema: params.schema, onStdout: (chunk) => Effect.sync(() => chunks.push(chunk)), projectEnvValues: params.projectEnvValues, + client: params.client, }); - return new TextDecoder().decode(concatChunks(chunks)); + return new TextDecoder().decode(Buffer.concat(chunks)); }); diff --git a/apps/cli/src/commands/migration/squash/squash.errors.ts b/apps/cli/src/commands/migration/squash/squash.errors.ts index 258bd60e7c..2509eec9ed 100644 --- a/apps/cli/src/commands/migration/squash/squash.errors.ts +++ b/apps/cli/src/commands/migration/squash/squash.errors.ts @@ -22,11 +22,13 @@ export class MigrationSquashMissingVersionError extends Data.TaggedError( } /** - * One of squash's three `pg_dump` containers exited non-zero. Matches the - * established `"error running container: exit " + code` text. + * One of squash's three `pg_dump` runs exited non-zero. Container dumps keep + * `"error running container: exit " + code`; native PATH dumps use + * `"error running pg_dump: exit " + code`. */ export class MigrationSquashDumpError extends Data.TaggedError("MigrationSquashDumpError")<{ readonly message: string; + readonly suggestion?: string; }> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { return actionability.dbConnection; diff --git a/apps/cli/src/commands/migration/squash/squash.handler.ts b/apps/cli/src/commands/migration/squash/squash.handler.ts index e15fd9a6e9..856dcd9c01 100644 --- a/apps/cli/src/commands/migration/squash/squash.handler.ts +++ b/apps/cli/src/commands/migration/squash/squash.handler.ts @@ -1,6 +1,7 @@ -import { Effect, FileSystem, Option, Path } from "effect"; +import { Effect, FileSystem, Option, Path, Predicate } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerType } from "effect/unstable/process/ChildProcessSpawner"; +import { resolveEphemeralPostgresRelease } from "@supabase/stack/effect"; import { cobraMutuallyExclusiveErrorMessage } from "../../../shared/cli/cobra-flag-groups.ts"; import { CliArgs } from "../../../shared/cli/cli-args.service.ts"; @@ -42,7 +43,15 @@ import type { ResolvedDbConfig } from "../../../command-internal/db-config.types import { DbConnection, type PgConnInput } from "../../../command-internal/db-connection.service.ts"; import { resolveDbTargetFlags } from "../../../command-internal/db-target-flags.ts"; import { DebugLogger } from "../../../command-internal/debug-logger.service.ts"; +import { DockerRunError } from "../../../command-internal/docker-run.errors.ts"; import { errorMessage, relativizeErrorMessage } from "../../../command-internal/error-message.ts"; +import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; +import { stackWithShadowDatabase } from "../../../command-internal/stack-shadow.ts"; +import { parsePostgresServerMajor } from "../../../command-internal/stack-local-database.ts"; +import { + dumpConnForHostClient, + rewriteDumpHostForToolContainer, +} from "../../../command-internal/postgres-client.run.ts"; import { applyMigrations, MigrationApplyError } from "../../../command-internal/migration-apply.ts"; import { INSERT_MIGRATION_VERSION, @@ -68,6 +77,7 @@ import { SQUASH_SEPARATOR_COMMENT, squashLineByLineDiff } from "./squash.diff.ts import { squashDumpSchema, squashDumpSchemaToString } from "./squash.dump.ts"; import { MigrationSquashBaselineError, + MigrationSquashDumpError, MigrationSquashMissingVersionError, MigrationSquashWriteError, } from "./squash.errors.ts"; @@ -89,7 +99,10 @@ const squashMigrations = Effect.fnUntraced(function* ( localInputs: LocalDbContainerInputs, toml: DbTomlValues, ) { - const resolvedShadowImage = yield* localInputs.resolvePostgresImage; + const stackBackend = (yield* currentStackBackend).kind === "stack"; + const resolvedShadowImage = stackBackend + ? "stack-ephemeral" + : yield* localInputs.resolvePostgresImage; const shadowInput = shadowRunInputFromLocalContainerInputs( localInputs, resolvedShadowImage, @@ -108,6 +121,125 @@ const squashMigrations = Effect.fnUntraced(function* ( // `pg_dump` container below uses; `squashDumpSchema` applies the registry mirror itself. const image = localInputs.bootstrapConfig.postgresImage; + if (stackBackend) { + const runtimeInfo = yield* RuntimeInfo; + return yield* stackWithShadowDatabase(shadowInput, (handle) => + Effect.scoped( + Effect.gen(function* () { + const stackConn: PgConnInput = { + host: handle.host, + port: handle.port, + user: "postgres", + password: toml.password, + database: "postgres", + }; + const networkIdFlag = yield* NetworkIdFlag; + const networkId = Option.getOrUndefined(networkIdFlag); + const dumpUsesHostNetwork = networkId === undefined || networkId.length === 0; + const nativeShadow = handle.runtime.kind === "native" && runtimeInfo.platform !== "win32"; + const expectedMajor = + parsePostgresServerMajor(handle.ephemeral.version) ?? toml.majorVersion; + const release = yield* resolveEphemeralPostgresRelease(handle.ephemeral.version).pipe( + Effect.orElseSucceed(() => undefined), + ); + const image = release?.image ?? localInputs.bootstrapConfig.postgresImage; + const dumpClient = nativeShadow + ? { + kind: "host" as const, + command: "pg_dump" as const, + expectedMajor, + } + : { kind: "container" as const }; + const dumpConn: PgConnInput = nativeShadow + ? dumpConnForHostClient(stackConn) + : { + ...stackConn, + host: rewriteDumpHostForToolContainer(handle.host, { + platform: runtimeInfo.platform, + usesHostNetwork: dumpUsesHostNetwork, + }), + }; + const session = yield* connectShadowDatabase(stackConn); + const before = yield* squashDumpSchemaToString({ + image, + conn: dumpConn, + schema: ["auth", "storage"], + projectEnvValues: localInputs.context.projectEnvValues, + client: dumpClient, + }); + yield* applyMigrations( + session, + fs, + path, + migrations, + (message) => new MigrationApplyError({ message }), + ); + const after = yield* squashDumpSchemaToString({ + image, + conn: dumpConn, + schema: ["auth", "storage"], + projectEnvValues: localInputs.context.projectEnvValues, + client: dumpClient, + }); + const targetPath = migrations[migrations.length - 1]!; + const targetRel = path.relative(workdir, targetPath); + yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(targetPath, { flag: "w", mode: 0o644 }).pipe( + Effect.mapError( + (cause) => + new MigrationSquashWriteError({ + message: `failed to open migration file: ${relativizeErrorMessage(errorMessage(cause), targetPath, targetRel)}`, + }), + ), + ); + yield* squashDumpSchema({ + image, + conn: dumpConn, + schema: [], + projectEnvValues: localInputs.context.projectEnvValues, + client: dumpClient, + onStdout: (chunk) => + file.writeAll(chunk).pipe( + Effect.mapError( + (cause) => + new MigrationSquashWriteError({ + message: `failed to copy docker logs: ${errorMessage(cause)}`, + }), + ), + ), + }); + const tail = SQUASH_SEPARATOR_COMMENT + squashLineByLineDiff(before, after); + yield* file.writeAll(new TextEncoder().encode(tail)).pipe( + Effect.mapError( + (cause) => + new MigrationSquashWriteError({ + message: `failed to write line: ${relativizeErrorMessage(errorMessage(cause), targetPath, targetRel)}`, + }), + ), + ); + }), + ); + }), + ).pipe( + Effect.catchIf( + (error): error is DockerRunError => + Predicate.isTagged(error, "DockerRunError") && + handle.runtime.kind === "native" && + runtimeInfo.platform === "win32", + (error) => + Effect.fail( + new MigrationSquashDumpError({ + message: error.message, + suggestion: + "Install Docker Desktop (or Git Bash) to squash a native stack on Windows.", + }), + ), + ), + ), + ); + } + yield* Effect.acquireUseRelease( createShadowDatabase(spawner, shadowInput), (handle) => diff --git a/apps/cli/src/commands/test/db/SIDE_EFFECTS.md b/apps/cli/src/commands/test/db/SIDE_EFFECTS.md index 674b0c256b..e90076c77a 100644 --- a/apps/cli/src/commands/test/db/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/test/db/SIDE_EFFECTS.md @@ -57,14 +57,14 @@ One-shot `docker run --rm `, where the image is `supabase/pg_pro ## Exit Codes -| Code | Condition | -| ---- | ---------------------------------------------------------------------------------------------------- | -| `0` | all pgTAP tests pass | -| `1` | `pg_prove` exits non-zero (test failures) — `error running container: exit N` | -| `1` | `pg_prove` ran no tests (`Result: NOTESTS`) — `no pgTAP tests found in `; Go exits `0` here | -| `1` | `--db-url` / `--linked` / `--local` set together (mutually exclusive) | -| `1` | database connection failure / pgTAP enable failure / docker failure / `--linked` auth or IPv6 errors | -| `1` | `--project-ref` set with a resolved target other than linked (see Notes) | +| Code | Condition | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------ | +| `0` | all pgTAP tests pass | +| `1` | `pg_prove` exits non-zero (test failures) — `error running container: exit N`, or `error running pg_prove: exit N` on a native stack | +| `1` | `pg_prove` ran no tests (`Result: NOTESTS`) — `no pgTAP tests found in `; Go exits `0` here | +| `1` | `--db-url` / `--linked` / `--local` set together (mutually exclusive) | +| `1` | database connection failure / pgTAP enable failure / docker failure / `--linked` auth or IPv6 errors | +| `1` | `--project-ref` set with a resolved target other than linked (see Notes) | ## Telemetry Events Fired diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index 9f750f3cfb..cc1fc0df5b 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -201,6 +201,7 @@ DbResetSeedFlagsError DbResetTargetFlagsError DbResetVersionFlagsError DbSetupError +DbStartFromBackupUnsupportedError DeclarativeApplyError DeclarativeCompatibilityError DeclarativeDiffError @@ -279,6 +280,7 @@ GenTypesWorkdirError GoChildExitError HealthCheckProbeError HealthCheckTimeoutError +HostPostgresClientError ImagePrepullError InitConfigExistsError InitExperimentalRequiredError @@ -421,6 +423,7 @@ PullOutputFlagUnsupportedError PullParentRefInvalidError PullUncommittedChangesError PullWorkdirError +ResetLocalDbFailedError ResetLocalDbNotRunningError ResetReplicationSlotsError RestartServicesError @@ -487,8 +490,11 @@ SsoUpdateMetadataFileError SsoUpdateNetworkError SsoUpdateNotFoundError SsoUpdateUnexpectedStatusError +StackCatalogSetupError StackConfigError +StackNativeEngineError StackRoutingError +StackRuntimeUnavailableError StartBackupVolumeExistsError StartConfigLoadError StartInvalidConfigError diff --git a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md new file mode 100644 index 0000000000..0badc71873 --- /dev/null +++ b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md @@ -0,0 +1,129 @@ +# 0025. Ephemeral Postgres for schema tooling + +**Status**: proposed +**Date**: 2026-09-09 + +## Problem Statement + +`db diff`, `db pull`, `db schema declarative`, and `migration squash` provision a throwaway +shadow Postgres, snapshot its platform baseline as a PGDATA tar, and compare it to a target. +That path today always uses the legacy Docker local database: compose container IDs, platform +SQL templates, and `db.shadow_port`. + +The managed stack runtime (`@supabase/stack`) is a different Postgres: slim-artifact init plus +a fixed role/JWT/`_supabase` bootstrap, native host `PGDATA` or a named volume, and no extra +database API. `[experimental].stack` currently switches top-level `start`/`stop`. With the flag +on, schema commands still inspect `supabase_db_` and shadow against the legacy +baseline, so `--local` diffs are wrong or impossible. + +A second Postgres **instance** is required. `CREATE DATABASE` on the live cluster is not +equivalent: declarative sync needs two independent servers, and the cache is a full PGDATA +snapshot. + +## Domain language + +- **Schema init**: the one-shot that mutates Postgres for an enabled capability that already has + a prepare/migrate process, without starting that capability’s long-running process. Not + activation, and not the CLI overlay. The throwaway compile is `database` plus the requested + one-shot names (live: auth, storage, realtime; analytics and pooler only when those one-shots + run). It never includes studio, mail, or functions. CLI `--exclude` does not change this set. +- **Overlay**: CLI session SQL after schema init: webhooks (`pg_net`), API default grants, vault + upsert, and `roles.sql`. +- **Activation**: starting a capability’s long-running process and listeners. Not schema init. + +## Decision + +### (a) Public `EphemeralPostgres` on `@supabase/stack` + +The package exposes a scoped, Supervisor-free Postgres cluster API (`createEphemeralPostgres`) +on both the Effect and Promise facades. It is not a stack identity: it does not appear in +`listStacks` / `discoverStacks`, and it does not persist `state.json` under the managed stacks +root. + +The cluster uses the same catalog artifact/image and the same bootstrap as a real stack +database. Callers own migrations, `roles.sql`, declarative SQL, and cache keys. + +Handle operations: loopback URL; `stop` (process/container down, data retained); `start` (from +existing data); `exportPgData` only while stopped; destroy on scope close. + +### (b) Snapshots are runtime-kind specific + +Native Postgres runs as the host user. Container snapshots preserve image uids. A Docker tar must +not restore onto native, and the reverse is also refused. The cache key includes `runtime.kind` +(and engine). Native export is a host-tree tar of `PGDATA`; container export tars the volume +through the catalog Postgres image. + +### (c) `[experimental].stack` covers the db/migration family + +`SUPABASE_EXPERIMENTAL_STACK` / `[experimental].stack` select the stack backend for `db` and +`migration` as well as `start`/`stop`. Flag off keeps the legacy Docker shadow and +`supabase_db_*` local target. Linked / `--db-url` targets are unchanged. Top-level `status` is +not switched. + +Shadow baseline for the stack backend is slim-init, stack bootstrap, schema init for the +platform trio (auth, storage, realtime), and the CLI overlay. Cache files use a distinct +`stack-shadow-baseline-*` namespace. Analytics and pooler stay off the shadow baseline. Schema +init never compiles studio, mail, or functions (those are not Postgres catalog one-shots). + +The stack backend requires the in-process pg-delta engine. Migra, pgAdmin, and +`--use-pg-schema` assume Docker networks or differ containers and are rejected for every +stack runtime. + +### (d) Native dump, test, and squash clients + +`db dump`, `db test`, and `migration squash` talk to published loopback credentials. On native +stacks they use PATH PostgreSQL clients except on Windows, where those commands run a one-shot +Docker `pg_dump` / `pg_prove` client against the published URL (`host.docker.internal`). The stack +stays native. If Docker is missing on that Windows path, the command fails and tells the user to +install Docker Desktop (or Git Bash). + +### (e) Studio does not require analytics + +Compose runs Studio when `[analytics] enabled = false`. Stack compile allows that pairing so +bare `stack start` matches Compose. Studio’s capability and workload graphs do not list analytics +as a hard dependency; logs UI stays off when analytics is off. This is independent of schema +init, which never compiles Studio. + +## Rationale + +Throwaway full stacks would pollute discovery, pull in a Supervisor, and still need a +pre-start PGDATA inject. Duplicating native spawn in the CLI would fork artifact and bootstrap +logic. A package-level cluster keeps one Postgres lifecycle for native and container while +leaving schema policy in the CLI. + +## Consequences + +### Positive + +- Native and Docker/Podman shadows share one API and the same slim baseline as `stack start`. +- Schema commands can target a running project stack through `credentials()` when the flag is on. + `credentials().database` is available whenever the database listener is assigned, including when + Auth is disabled. `credentials().api` is absent when Auth is off. Overlay and `--local` keep + calling `credentials()`. There is no second RPC, and the CLI does not read secret slots. +- `resetDatabase` wipes Postgres without destroying the stack identity, so `db reset --local` and declarative `--apply` stay on the stack backend. +- Legacy Docker behavior is unchanged when the flag is off. +- Windows native stacks can dump and squash without PostgreSQL client tools on PATH. + +### Negative + +- Cache tars cannot be shared across native and container runtimes. +- Migra/pgAdmin remain unavailable on stack backends. +- Windows native dump/test/squash need a working Docker client even though Postgres itself is native. + +## Alternatives Considered + +1. **Database-only throwaway stacks** via `createStack`/`destroy`: extra Supervisor and + registry identity for a tooling cluster; cache restore still needs a data inject. +2. **CLI-owned spawn**: Docker shadows with the slim image, CLI-spawned native binary. Forks + catalog/bootstrap from the runtime package. +3. **`CREATE DATABASE` on the live cluster**: cannot snapshot independently or run two + declarative plan servers. + +## Related Decisions + +- ADR 0017: Simplified managed stack architecture + +## See Also + +- [`packages/stack/README.md`](../../packages/stack/README.md) +- [`apps/cli/docs/stack-commands.md`](../../apps/cli/docs/stack-commands.md) diff --git a/docs/adr/README.md b/docs/adr/README.md index 4a99680ad5..083f7fb36e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -65,6 +65,7 @@ When an ADR becomes outdated, mark it as `deprecated` or reference the supersedi | 0022 | [Config Diff Classification and Managed Surface](0022-config-diff-classification-and-managed-surface.md) | accepted | | 0023 | [Config Pull Write Strategy and Scope Resolution](0023-config-pull-write-strategy-and-scope-resolution.md) | accepted | | 0024 | [Top-Level `pull` Orchestration](0024-top-level-pull-orchestration.md) | accepted | +| 0025 | [Ephemeral Postgres for Schema Tooling](0025-ephemeral-postgres-for-schema-tooling.md) | proposed | ## Template diff --git a/packages/config/src/experimental.ts b/packages/config/src/experimental.ts index 36c2e5dbb2..3bb116813d 100644 --- a/packages/config/src/experimental.ts +++ b/packages/config/src/experimental.ts @@ -40,7 +40,8 @@ export const experimental = Schema.Struct({ ), stack: Schema.optionalKey( Schema.Boolean.annotate({ - description: "Use the new local stack backend for top-level start and stop commands.", + description: + "Use the new local stack backend for top-level start and stop commands, and for the db and migration command families.", tags, }), ), diff --git a/packages/stack/README.md b/packages/stack/README.md index 2690732e2a..5f8b05ce00 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -25,8 +25,8 @@ PostgreSQL by default; capabilities configured as eager join its startup depende The remaining lazy capabilities activate through the stack's listeners on demand for the current running session. The Effect API's `excludeStackCapabilities` helper disables requested optional capabilities and -their dependents in an in-memory config. Excluding `rest` or `analytics` also disables `studio`, -while the database remains required. The project config is unchanged, and runtime listeners are +their dependents in an in-memory config. Excluding `rest` also disables `studio`; excluding +`analytics` does not. The database remains required. The project config is unchanged, and runtime listeners are created only for enabled capability routes. Native workloads have a two-minute readiness budget to allow cold starts to load shared libraries; container workloads retain a 30-second budget, and PostgreSQL uses its configured `health_timeout`. @@ -112,7 +112,15 @@ Each capability may opt into eager activation in `StackConfig`; omitted settings non-PostgreSQL capability lazy. Prepared artifacts are not automatically pruned. `followLogs(...)` provides filterable live entries through a stateless client-polled cursor. -Database reset is intentionally outside the current API. Applying migrations, declarative schemas, -and seeds remains the caller's responsibility. The runtime bootstrap only reconciles the `_realtime` -schema owner, closed database role passwords, and JWT settings in one transaction; the slim database -artifact owns its initialization and migrations. +`resetDatabase()` wipes Postgres data only: identity, ports, secrets, logs, and storage volumes +stay. The database is started and bootstrapped before return. Applying migrations, declarative +schemas, and seeds remains the caller's responsibility. The runtime bootstrap only reconciles the +`_realtime` schema owner, closed database role passwords, and JWT settings in one transaction; the +slim database artifact owns its initialization and migrations. + +`createEphemeralPostgres` is a scoped, Supervisor-free Postgres cluster for schema tooling. It uses +the same catalog artifact and bootstrap as a stack database, is not registered in `listStacks` / +`discoverStacks`, and destroys its data directory or volume when the Effect scope closes. The +Promise facade returns a handle with explicit `destroy()`. Callers own migrations and PGDATA +cache keys. `exportPgData` is valid only while the cluster is stopped; native and container snapshots +are not interchangeable. diff --git a/packages/stack/src/control/StackRpc.ts b/packages/stack/src/control/StackRpc.ts index 1fa472d938..97ea96a9d0 100644 --- a/packages/stack/src/control/StackRpc.ts +++ b/packages/stack/src/control/StackRpc.ts @@ -9,7 +9,7 @@ import { StackStatusSchema } from "../public/Status.ts"; import { STACK_ERROR_TAGS } from "../public/Errors.ts"; /** Pinned release identifier used to detect incompatible live owners. */ -export const STACK_RPC_RELEASE = "stack-rpc-v1@0.1.0" as const; +export const STACK_RPC_RELEASE = "stack-rpc-v1@0.2.0" as const; const StackRpcErrorTagSchema = Schema.Literals([...STACK_ERROR_TAGS] as const); @@ -31,6 +31,10 @@ const StackRpc = { error: StackRpcErrorSchema, }), destroy: Rpc.make("destroy", { success: Schema.Void, error: StackRpcErrorSchema }), + resetDatabase: Rpc.make("resetDatabase", { + success: StackStatusSchema, + error: StackRpcErrorSchema, + }), logs: Rpc.make("logs", { payload: LogQuerySchema, success: StackLogBatchSchema, @@ -43,6 +47,7 @@ export const StackRpcGroup = RpcGroup.make( StackRpc.credentials, StackRpc.start, StackRpc.destroy, + StackRpc.resetDatabase, StackRpc.logs, ); type StackRpcDefinitions = RpcGroup.Rpcs; diff --git a/packages/stack/src/control/control-transport.integration.test.ts b/packages/stack/src/control/control-transport.integration.test.ts index 3aff07cc9d..fa1793550b 100644 --- a/packages/stack/src/control/control-transport.integration.test.ts +++ b/packages/stack/src/control/control-transport.integration.test.ts @@ -42,7 +42,7 @@ import { encodePreface, MaintenanceProtocolError, } from "./MaintenanceProtocol.ts"; -import type { StackRpcError, StackRpcHandlers } from "./StackRpc.ts"; +import { STACK_RPC_RELEASE, type StackRpcError, type StackRpcHandlers } from "./StackRpc.ts"; interface ServerOverrides { readonly rpcHandlers?: Partial; @@ -147,6 +147,7 @@ const withServer = ( }), start: () => Effect.succeed(status), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(status), logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), }; const defaultMaintenanceHandlers: MaintenanceHandlers = { @@ -155,7 +156,7 @@ const withServer = ( op: "probe", stackId, ownerSessionId, - rpcRelease: "stack-rpc-v1@0.1.0", + rpcRelease: STACK_RPC_RELEASE, }), stop: Effect.succeed({ ok: true, op: "stop" }), }; @@ -456,7 +457,7 @@ describe("control transport", () => { Effect.gen(function* () { const preface = encodePreface({ kind: "rpc", - release: "stack-rpc-v1@0.1.0", + release: STACK_RPC_RELEASE, stackId, ownerSessionId, }); @@ -503,7 +504,7 @@ describe("control transport", () => { Effect.gen(function* () { const preface = encodePreface({ kind: "rpc", - release: "stack-rpc-v1@0.1.0", + release: STACK_RPC_RELEASE, stackId, ownerSessionId, }); @@ -613,7 +614,7 @@ describe("control transport", () => { const ownerSessionId = "session"; const preface = encodePreface({ kind: "rpc", - release: "stack-rpc-v1@0.1.0", + release: STACK_RPC_RELEASE, stackId, ownerSessionId, }); @@ -630,7 +631,7 @@ describe("control transport", () => { Effect.gen(function* () { const preface = encodePreface({ kind: "rpc", - release: "stack-rpc-v1@0.1.0", + release: STACK_RPC_RELEASE, stackId: "b".repeat(64), ownerSessionId, }); @@ -647,7 +648,7 @@ describe("control transport", () => { Effect.gen(function* () { const preface = encodePreface({ kind: "rpc", - release: "stack-rpc-v1@0.1.0", + release: STACK_RPC_RELEASE, stackId, ownerSessionId: "stale-session", }); @@ -674,6 +675,28 @@ describe("control transport", () => { ), ); + it.live("rejects an older stack RPC release with an upgrade error", () => + withServer(({ endpoint, stackId, ownerSessionId }) => + Effect.gen(function* () { + const preface = encodePreface({ + kind: "rpc", + release: "stack-rpc-v1@0.1.0", + stackId, + ownerSessionId, + }); + const invalid = concatBytes(new Uint8Array([0, 0, 0, 1]), new Uint8Array([0xff])); + const response = yield* sendRawAndReadFrame(endpoint, concatBytes(preface, invalid)); + expect(response).toMatchObject({ + ok: false, + error: { + tag: "unsupported-release", + message: `Incompatible Stack RPC release; expected ${STACK_RPC_RELEASE}, received stack-rpc-v1@0.1.0`, + }, + }); + }), + ), + ); + it.live("dispatches exactly one maintenance request on a connection", () => { return withServer( ({ endpoint, stackId, ownerSessionId, stopCalls }) => diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index 4d863bb877..4d46519f33 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -5,12 +5,15 @@ export { listStacks, discoverStacks, inspectStack, + createEphemeralPostgres, } from "./public/PromiseStack.ts"; export type { PromiseStack, PromiseStackConfig, PromiseStartStackOptions, PromisePrepareStackOptions, + PromiseCreateEphemeralPostgresOptions, + PromiseEphemeralPostgres, CreateStackOptions, FindStackOptions, ListStacksOptions, diff --git a/packages/stack/src/model/DatabaseBootstrap.ts b/packages/stack/src/model/DatabaseBootstrap.ts index 8262d5a958..0a32a89ddf 100644 --- a/packages/stack/src/model/DatabaseBootstrap.ts +++ b/packages/stack/src/model/DatabaseBootstrap.ts @@ -15,13 +15,16 @@ const DATABASE_BOOTSTRAP_ROLES = [ ] as const; type DatabaseBootstrapRole = (typeof DATABASE_BOOTSTRAP_ROLES)[number]; +export const JWT_SECRET_SETTING = "app.settings.jwt_secret" as const; +const JWT_EXP_SETTING = "app.settings.jwt_exp" as const; + type DatabaseBootstrapSetting = | { - readonly name: "app.settings.jwt_secret"; + readonly name: typeof JWT_SECRET_SETTING; readonly value: Redacted.Redacted; } | { - readonly name: "app.settings.jwt_exp"; + readonly name: typeof JWT_EXP_SETTING; readonly value: number; }; @@ -74,6 +77,22 @@ const REALTIME_SCHEMA_STATEMENT = "CREATE SCHEMA IF NOT EXISTS _realtime;\nALTER SCHEMA _realtime OWNER TO postgres;"; const ADVISORY_LOCK_STATEMENT = `SELECT pg_advisory_xact_lock(hashtext('supabase_internal.bootstrap'));`; +/** Private database created outside the bootstrap transaction. */ +export const INTERNAL_DATABASE = "_supabase"; +/** Service-owned schemas created in the private database. */ +export const INTERNAL_SCHEMAS = ["_analytics", "_supavisor"] as const; + +/** Cache/key material for the managed bootstrap. */ +export const databaseBootstrapIdentity = [ + ...DATABASE_BOOTSTRAP_ROLES, + ADVISORY_LOCK_STATEMENT, + REALTIME_SCHEMA_STATEMENT, + JWT_SECRET_SETTING, + JWT_EXP_SETTING, + INTERNAL_DATABASE, + ...INTERNAL_SCHEMAS, +].join("\n"); + const statementError = (error: DatabaseBootstrapError, statement: string) => new DatabaseBootstrapError({ message: error.message, @@ -106,8 +125,8 @@ export const runDatabaseBootstrap = ( ); yield* transaction .setDatabaseSettings([ - { name: "app.settings.jwt_secret", value: options.jwtSecret }, - { name: "app.settings.jwt_exp", value: options.jwtExpiry }, + { name: JWT_SECRET_SETTING, value: options.jwtSecret }, + { name: JWT_EXP_SETTING, value: options.jwtExpiry }, ]) .pipe( Effect.mapError( diff --git a/packages/stack/src/model/capabilities/studio.ts b/packages/stack/src/model/capabilities/studio.ts index bbc3abce39..6d58f8d2ca 100644 --- a/packages/stack/src/model/capabilities/studio.ts +++ b/packages/stack/src/model/capabilities/studio.ts @@ -17,11 +17,11 @@ export const StudioModule: CapabilityModule = { defaultEnabled: true, defaultActivation: "lazy", defaultVersion: version, - dependencies: ["rest", "analytics"], + dependencies: ["rest"], releases: { [version]: release(version, [ workload("studio", "studio", { - dependencies: ["studio:pgmeta", "analytics:analytics"], + dependencies: ["studio:pgmeta"], readiness: { portField: "studio" }, }), workload("pgmeta", "studio", { diff --git a/packages/stack/src/model/compiler.integration.test.ts b/packages/stack/src/model/compiler.integration.test.ts index ff6dc6d62e..a886e0220f 100644 --- a/packages/stack/src/model/compiler.integration.test.ts +++ b/packages/stack/src/model/compiler.integration.test.ts @@ -41,8 +41,8 @@ describe("closed capability compiler", () => { const result = yield* compile(excluded); expect(result.definition.capabilities[name].enabled).toBe(false); if (name !== "rest") expect(result.definition.capabilities.rest.settings.max_rows).toBe(42); - if (name === "rest" || name === "analytics") - expect(result.definition.capabilities.studio.enabled).toBe(false); + if (name === "rest") expect(result.definition.capabilities.studio.enabled).toBe(false); + if (name === "analytics") expect(result.definition.capabilities.studio.enabled).toBe(true); } const combined = excludeStackCapabilities({}, ["rest", "analytics"]); const result = yield* compile(combined); @@ -54,6 +54,17 @@ describe("closed capability compiler", () => { const studioExcluded = yield* compile(excludeStackCapabilities({}, ["studio"])); expect(studioExcluded.definition.capabilities.rest.enabled).toBe(true); expect(studioExcluded.definition.capabilities.analytics.enabled).toBe(true); + const analyticsOff = yield* compile({ + capabilities: { analytics: { enabled: false } }, + }); + expect(analyticsOff.definition.capabilities.studio.enabled).toBe(true); + expect(analyticsOff.definition.capabilities.analytics.enabled).toBe(false); + expect(analyticsOff.executionPlan.workloads.some(({ id }) => id === "studio:studio")).toBe( + true, + ); + expect( + analyticsOff.executionPlan.workloads.some(({ id }) => id === "analytics:analytics"), + ).toBe(false); }), ); diff --git a/packages/stack/src/public/Credentials.ts b/packages/stack/src/public/Credentials.ts index f75b47c31e..8b78a3e631 100644 --- a/packages/stack/src/public/Credentials.ts +++ b/packages/stack/src/public/Credentials.ts @@ -22,7 +22,7 @@ const EffectStorageCredentialsSchema = Schema.Struct({ export const EffectStackCredentialsSchema = Schema.Struct({ database: EffectDatabaseCredentialsSchema, - api: EffectApiCredentialsSchema, + api: Schema.optionalKey(EffectApiCredentialsSchema), storage: Schema.optionalKey(EffectStorageCredentialsSchema), }); export interface EffectStackCredentials { @@ -30,7 +30,7 @@ export interface EffectStackCredentials { readonly url: Redacted.Redacted; readonly password: Redacted.Redacted; }; - readonly api: { + readonly api?: { readonly publishableKey: string; readonly secretKey: Redacted.Redacted; readonly anonJwt: string; @@ -49,12 +49,14 @@ export const PromiseStackCredentialsSchema = Schema.Struct({ url: Schema.String, password: Schema.String, }), - api: Schema.Struct({ - publishableKey: Schema.String, - secretKey: Schema.String, - anonJwt: Schema.String, - serviceRoleJwt: Schema.String, - }), + api: Schema.optionalKey( + Schema.Struct({ + publishableKey: Schema.String, + secretKey: Schema.String, + anonJwt: Schema.String, + serviceRoleJwt: Schema.String, + }), + ), storage: Schema.optionalKey( Schema.Struct({ endpoint: Schema.String, diff --git a/packages/stack/src/public/EffectStack.ts b/packages/stack/src/public/EffectStack.ts index 252c441936..4f9f57d5d5 100644 --- a/packages/stack/src/public/EffectStack.ts +++ b/packages/stack/src/public/EffectStack.ts @@ -72,6 +72,8 @@ import { PortUnavailableError, GatewayActivationError, InvalidLogCursorError, + EphemeralPostgresError, + RequiresActivatedProcessError, type CreateStackError, type OpenStackError, type StackDiscoveryError, @@ -82,6 +84,7 @@ import { type StackStopError, type StackLogsError, type DestroyStackError, + type ResetDatabaseError, type StackError, type StackErrorTag, isStackError, @@ -93,6 +96,7 @@ import { STACK_STOP_ERROR_TAGS, STACK_LOGS_ERROR_TAGS, DESTROY_STACK_ERROR_TAGS, + RESET_DATABASE_ERROR_TAGS, } from "./Errors.ts"; import { ownerLockExists, @@ -113,10 +117,9 @@ import { } from "../supervisor/Launcher.ts"; import { ContainerEngineResolver, - defaultContainerEngineResolver, + selectDefaultRuntime, type ContainerEngineResolverShape, } from "../runtime/ContainerEngineResolver.ts"; -import type { ContainerEngineFailure } from "../runtime/ContainerEngine.ts"; import { statusFor } from "../supervisor/StatusProjection.ts"; import { EMPTY_LOG_CURSOR, readRetainedLogs, selectLogBatch } from "../supervisor/LogStore.ts"; import { @@ -151,24 +154,6 @@ export interface PreparedCapability { readonly outcome: "cached" | "downloaded" | "pulled"; } -const selectDefaultRuntime = ( - resolver: ContainerEngineResolverShape | undefined, -): Effect.Effect => { - return (resolver ?? defaultContainerEngineResolver).isInstalled("docker").pipe( - Effect.map((installed): StackRuntime => - installed ? { kind: "container", engine: "docker" } : { kind: "native" }, - ), - Effect.mapError( - (error: ContainerEngineFailure) => - new ContainerEngineError({ - engine: "docker", - message: `Unable to determine whether Docker is installed: ${error.message}`, - cause: error, - }), - ), - ); -}; - export interface PrepareStackResult { readonly capabilities: ReadonlyArray; } @@ -183,6 +168,7 @@ export interface EffectStack { readonly start: (options?: StartStackOptions) => Effect.Effect; readonly stop: Effect.Effect; readonly destroy: Effect.Effect; + readonly resetDatabase: Effect.Effect; readonly logs: (query?: LogQuery) => Effect.Effect; readonly followLogs: (query?: LogQuery) => Stream.Stream; } @@ -245,6 +231,9 @@ const stackErrorFactories = { StackCleanupError: (message: string) => new StackCleanupError({ message }), ContainerEngineError: (message: string) => new ContainerEngineError({ message }), StackDestructionError: (message: string) => new StackDestructionError({ message }), + EphemeralPostgresError: (message: string) => new EphemeralPostgresError({ message }), + RequiresActivatedProcessError: (message: string) => + new RequiresActivatedProcessError({ message, capability: "unknown" }), } satisfies Record StackError>; const isOwnerUnreachable = (error: unknown): boolean => @@ -306,6 +295,12 @@ const logsError = (error: ControlError): StackLogsError => narrowError(error, STACK_LOGS_ERROR_TAGS, (message) => new StackStateInvalidError({ message })); const destroyError = (error: ControlError): DestroyStackError => narrowError(error, DESTROY_STACK_ERROR_TAGS, (message) => new StackDestructionError({ message })); +const resetDatabaseError = (error: ControlError): ResetDatabaseError => + narrowError( + error, + RESET_DATABASE_ERROR_TAGS, + (message) => new StackStateInvalidError({ message }), + ); /** Internal control-transport seam used by public lifecycle integration tests. */ export interface HandleDependencies { @@ -579,6 +574,30 @@ export const makeHandle = (id: StackId, options: HandleDependencies): Effect.Eff }), ), ); + const resetDatabase: Effect.Effect = Effect.suspend( + (): Effect.Effect => + invoke((rpc) => rpc.resetDatabase(undefined), resetDatabaseError).pipe( + Effect.catchTag("StackOwnershipConflictError", (ownershipError) => { + const offline: Effect.Effect = options.readOfflineState.pipe( + Effect.mapError(resetDatabaseError), + Effect.flatMap((state): Effect.Effect => + Option.isNone(state) + ? Effect.fail(stackNotFound()) + : isStoppedState(state.value) + ? Effect.fail( + new StackNotRunningError({ + stackId: id, + message: "Stack is not running", + }), + ) + : Effect.fail(ownershipError), + ), + Effect.catchTag("StackOwnershipConflictError", () => Effect.fail(ownershipError)), + ); + return offline; + }), + ), + ); const start = (startOptions?: StartStackOptions) => { return invoke( (rpc) => @@ -687,6 +706,7 @@ export const makeHandle = (id: StackId, options: HandleDependencies): Effect.Eff start, stop, destroy, + resetDatabase, logs, followLogs: (query) => Stream.paginate({ cursor: query?.cursor, first: true }, ({ cursor, first }) => { diff --git a/packages/stack/src/public/EphemeralPostgres.ts b/packages/stack/src/public/EphemeralPostgres.ts new file mode 100644 index 0000000000..6fca7d5a9f --- /dev/null +++ b/packages/stack/src/public/EphemeralPostgres.ts @@ -0,0 +1,84 @@ +import { Crypto, Effect, FileSystem, Path, Redacted, Scope } from "effect"; +import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +import { DatabaseModule } from "../model/capabilities/database.ts"; +import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; +import { createEphemeralPostgresCluster } from "../runtime/EphemeralPostgres.ts"; +import type { EphemeralPostgresCreateError, EphemeralPostgresError } from "./Errors.ts"; +import { StackVersionUnsupportedError } from "./Errors.ts"; +import type { StackRuntime, StackRuntimePreference } from "./Runtime.ts"; + +export interface EphemeralPostgresSettings { + readonly [key: string]: string | number | boolean | undefined; +} + +export interface CreateEphemeralPostgresOptions { + /** Omitted preference uses Docker when installed, otherwise native. */ + readonly runtime?: StackRuntimePreference; + /** Exact catalog release or major selector such as `"17"`. */ + readonly version?: string; + readonly port?: number; + readonly databasePassword: Redacted.Redacted; + readonly jwtSecret: Redacted.Redacted; + readonly jwtExpiry?: number; + readonly postgresSettings?: EphemeralPostgresSettings; + readonly healthTimeout?: string; + /** Stopped-cluster PGDATA tar to restore before the first start. */ + readonly restoreFrom?: string; +} + +export interface EphemeralPostgresRelease { + readonly version: string; + readonly image: string; +} + +export type EphemeralPostgresServices = + | ChildProcessSpawnerService + | Scope.Scope + | FileSystem.FileSystem + | Path.Path; + +export interface EffectEphemeralPostgres { + readonly host: string; + readonly port: number; + readonly version: string; + readonly runtime: StackRuntime; + /** Catalog identity hashed into CLI shadow-cache keys. */ + readonly artifactIdentity: string; + readonly url: Redacted.Redacted; + /** Container network id so schema-init one-shots can join and dial `supabase-database:5432`. */ + readonly networkId?: string; + readonly start: Effect.Effect; + readonly stop: Effect.Effect; + readonly exportPgData: ( + tarPath: string, + ) => Effect.Effect; +} + +/** Resolves a Postgres catalog release the same way stack compilation does. */ +export const resolveEphemeralPostgresRelease = ( + version?: string, +): Effect.Effect => { + const requested = version ?? DatabaseModule.defaultVersion; + const selected = DatabaseModule.releases[requested]; + const release = + selected === undefined + ? catalogReleaseFor("database:database", requested) + : catalogReleaseFor("database:database", selected.version); + if (release === undefined) + return Effect.fail( + new StackVersionUnsupportedError({ + message: `Unsupported PostgreSQL version ${requested}`, + version: requested, + capability: "database", + }), + ); + return Effect.succeed({ version: release.version, image: release.containerImage }); +}; + +export const createEphemeralPostgres = ( + options: CreateEphemeralPostgresOptions, +): Effect.Effect< + EffectEphemeralPostgres, + EphemeralPostgresCreateError, + Scope.Scope | FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService +> => createEphemeralPostgresCluster(options); diff --git a/packages/stack/src/public/Errors.ts b/packages/stack/src/public/Errors.ts index cb56aabc90..d17e09dc7e 100644 --- a/packages/stack/src/public/Errors.ts +++ b/packages/stack/src/public/Errors.ts @@ -136,6 +136,22 @@ export class ContainerEngineError extends Data.TaggedError("ContainerEngineError ErrorFields & { readonly engine?: ContainerEngineKind } > {} export class StackDestructionError extends Data.TaggedError("StackDestructionError") {} +export class EphemeralPostgresError extends Data.TaggedError("EphemeralPostgresError")< + ErrorFields & { + readonly reason?: + | "not-stopped" + | "not-running" + | "snapshot" + | "restore-mismatch" + | "bootstrap" + | "destroy"; + readonly path?: string; + readonly version?: string; + } +> {} +export class RequiresActivatedProcessError extends Data.TaggedError( + "RequiresActivatedProcessError", +) {} /** Stable wire tags for errors produced by the managed stack runtime. */ export const STACK_ERROR_TAGS = [ @@ -165,6 +181,8 @@ export const STACK_ERROR_TAGS = [ "StackCleanupError", "ContainerEngineError", "StackDestructionError", + "EphemeralPostgresError", + "RequiresActivatedProcessError", ] as const; export type StackErrorTag = (typeof STACK_ERROR_TAGS)[number]; @@ -198,7 +216,9 @@ export type StackError = | StackRuntimeError | StackCleanupError | ContainerEngineError - | StackDestructionError; + | StackDestructionError + | EphemeralPostgresError + | RequiresActivatedProcessError; export const isStackError = (value: unknown): value is StackError => Predicate.hasProperty(value, "_tag") && @@ -325,3 +345,40 @@ export const DESTROY_STACK_ERROR_TAGS = [ "StackUpgradeRequiredError", ] as const satisfies ReadonlyArray; export type DestroyStackError = ErrorByTag<(typeof DESTROY_STACK_ERROR_TAGS)[number]>; + +export const RESET_DATABASE_ERROR_TAGS = [ + "StackNotFoundError", + ...STACK_START_ERROR_TAGS, +] as const satisfies ReadonlyArray; +export type ResetDatabaseError = ErrorByTag<(typeof RESET_DATABASE_ERROR_TAGS)[number]>; + +export const EPHEMERAL_POSTGRES_ERROR_TAGS = [ + "EphemeralPostgresError", + "StackVersionUnsupportedError", + "PortUnavailableError", + "StackPreparationError", + "ArtifactIntegrityError", + "ContainerPullError", + "ContainerEngineError", +] as const satisfies ReadonlyArray; +export type EphemeralPostgresCreateError = ErrorByTag< + (typeof EPHEMERAL_POSTGRES_ERROR_TAGS)[number] +>; + +export const SCHEMA_INIT_ERROR_TAGS = [ + "RequiresActivatedProcessError", + "InvalidStackConfigError", + "StackVersionUnsupportedError", + "InvalidProjectRootError", + "InvalidStackIdentityError", + "StackPreparationError", + "ArtifactIntegrityError", + "ContainerPullError", + "ContainerEngineError", + "StackSecretMismatchError", + "InvalidJwtSigningMaterialError", + "StackRuntimeError", + "StackMustBeStoppedError", + "StackStateInvalidError", +] as const satisfies ReadonlyArray; +export type SchemaInitError = ErrorByTag<(typeof SCHEMA_INIT_ERROR_TAGS)[number]>; diff --git a/packages/stack/src/public/PromiseStack.ts b/packages/stack/src/public/PromiseStack.ts index ee00acdeb9..24b7b65816 100644 --- a/packages/stack/src/public/PromiseStack.ts +++ b/packages/stack/src/public/PromiseStack.ts @@ -1,5 +1,17 @@ import { NodeServices } from "@effect/platform-node"; -import { Crypto, Effect, FileSystem, Layer, Option, Path, Redacted, Schema, Stream } from "effect"; +import { + Crypto, + Effect, + Exit, + FileSystem, + Layer, + Option, + Path, + Redacted, + Schema, + Scope, + Stream, +} from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { createStack as createEffectStack, @@ -26,6 +38,11 @@ import type { StackId } from "./StackId.ts"; import type { PreparedCapability, PrepareStackResult } from "./EffectStack.ts"; import { InvalidStackConfigError } from "./Errors.ts"; import { StackRuntimeEnvironment, type StackRuntimeEnvironmentValue } from "../state/Ownership.ts"; +import { + createEphemeralPostgres as createEffectEphemeralPostgres, + type CreateEphemeralPostgresOptions, +} from "./EphemeralPostgres.ts"; +import type { StackRuntime } from "./Runtime.ts"; /** Recursively replaces Effect `Redacted` leaves with their plain value. */ type Unredacted = @@ -53,10 +70,32 @@ export interface PromiseStack { readonly start: (options?: PromiseStartStackOptions) => Promise; readonly stop: () => Promise; readonly destroy: () => Promise; + readonly resetDatabase: () => Promise; readonly logs: (query?: LogQuery) => Promise; readonly followLogs: (query?: LogQuery) => AsyncIterable; } +export type PromiseCreateEphemeralPostgresOptions = Omit< + CreateEphemeralPostgresOptions, + "databasePassword" | "jwtSecret" +> & { + readonly databasePassword: string; + readonly jwtSecret: string; +}; + +export interface PromiseEphemeralPostgres { + readonly host: string; + readonly port: number; + readonly version: string; + readonly runtime: StackRuntime; + readonly artifactIdentity: string; + readonly url: string; + readonly start: () => Promise; + readonly stop: () => Promise; + readonly exportPgData: (tarPath: string) => Promise; + readonly destroy: () => Promise; +} + interface PromiseStackApi { readonly createStack: (options: CreateStackOptions) => Promise; readonly openStack: (id: StackId) => Promise; @@ -64,6 +103,9 @@ interface PromiseStackApi { readonly listStacks: (options?: ListStacksOptions) => Promise>; readonly discoverStacks: (options?: ListStacksOptions) => Promise; readonly inspectStack: (id: StackId) => Promise; + readonly createEphemeralPostgres: ( + options: PromiseCreateEphemeralPostgresOptions, + ) => Promise; } type PlatformLayer = typeof NodeServices.layer; @@ -149,6 +191,7 @@ export const adaptEffectStack = (effectStack: EffectStack): PromiseStack => { ), stop: () => invoke(effectStack.stop), destroy: () => invoke(effectStack.destroy), + resetDatabase: () => invoke(effectStack.resetDatabase), logs: (query) => invoke(effectStack.logs(query)), followLogs: (query) => adaptStream(effectStack.followLogs(query)), }; @@ -176,6 +219,37 @@ export const makePromiseApi = ( listStacks: (options) => run(listEffectStacks(options)), discoverStacks: (options) => run(discoverEffectStacks(options)), inspectStack: (id) => run(inspectEffectStack(id)), + createEphemeralPostgres: (options) => + run( + Effect.gen(function* () { + const scope = yield* Scope.make(); + const handle = yield* createEffectEphemeralPostgres({ + ...options, + databasePassword: Redacted.make(options.databasePassword), + jwtSecret: Redacted.make(options.jwtSecret), + }).pipe( + Effect.provideService(Scope.Scope, scope), + Effect.onError(() => Scope.close(scope, Exit.void).pipe(Effect.ignore)), + ); + return { handle, scope }; + }), + ).then(({ handle, scope }) => { + const runInScope = ( + effect: Effect.Effect, + ): Promise => run(effect.pipe(Effect.provideService(Scope.Scope, scope))); + return { + host: handle.host, + port: handle.port, + version: handle.version, + runtime: handle.runtime, + artifactIdentity: handle.artifactIdentity, + url: Redacted.value(handle.url), + start: () => runInScope(handle.start), + stop: () => runInScope(handle.stop), + exportPgData: (tarPath: string) => runInScope(handle.exportPgData(tarPath)), + destroy: () => run(Scope.close(scope, Exit.void)), + }; + }), }; }; @@ -186,6 +260,7 @@ export const findStack = defaultApi.findStack; export const listStacks = defaultApi.listStacks; export const discoverStacks = defaultApi.discoverStacks; export const inspectStack = defaultApi.inspectStack; +export const createEphemeralPostgres = defaultApi.createEphemeralPostgres; export type { CreateStackOptions, diff --git a/packages/stack/src/public/SchemaInit.ts b/packages/stack/src/public/SchemaInit.ts new file mode 100644 index 0000000000..21d524f425 --- /dev/null +++ b/packages/stack/src/public/SchemaInit.ts @@ -0,0 +1,67 @@ +import { Crypto, Effect, FileSystem, Path, Redacted } from "effect"; +import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +import type { SchemaInitError } from "./Errors.ts"; +import type { StackConfig } from "./Config.ts"; +import type { StackRuntime } from "./Runtime.ts"; +import type { StackId } from "./StackId.ts"; +import { schemaInitWorkloads, schemaInitArtifactIdentity } from "../runtime/SchemaInit.ts"; +import type { ContainerEngine } from "../runtime/ContainerEngine.ts"; +import type { RuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; + +export { schemaInitArtifactIdentity }; + +export const SCHEMA_INIT_CAPABILITY_NAMES = [ + "auth", + "storage", + "realtime", + "analytics", + "pooler", +] as const; +export type SchemaInitCapabilityName = (typeof SCHEMA_INIT_CAPABILITY_NAMES)[number]; + +export interface SchemaInitSecrets { + readonly databasePassword: Redacted.Redacted; + readonly jwtSecret?: Redacted.Redacted; +} + +interface SchemaInitTargetBase { + readonly projectRoot: string; + readonly runtime: StackRuntime; + readonly config: StackConfig; + readonly databaseUrl: string; + readonly secrets: SchemaInitSecrets; +} + +export interface SchemaInitLiveTarget extends SchemaInitTargetBase { + readonly kind: "live"; + readonly stackId: StackId; +} + +export interface SchemaInitEphemeralTarget extends SchemaInitTargetBase { + readonly kind: "ephemeral"; + /** Container network of the throwaway Postgres cluster; one-shots join it and dial `supabase-database:5432`. */ + readonly networkId?: string; +} + +export type SchemaInitTarget = SchemaInitLiveTarget | SchemaInitEphemeralTarget; + +export interface SchemaInitOptions { + readonly containerEngine?: ContainerEngine; + readonly artifactPreparer?: RuntimeArtifactPreparer; + /** Host OS for Linux extra hosts so `host.docker.internal` resolves; DNS only, not URL rewrite. */ + readonly platform?: string; +} + +export type SchemaInitServices = + | ChildProcessSpawnerService + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto; + +/** Runs service-owned one-shots against a target Postgres without activating long-running processes. */ +export const schemaInit = ( + names: ReadonlyArray, + target: SchemaInitTarget, + options: SchemaInitOptions = {}, +): Effect.Effect => + schemaInitWorkloads(names, target, options); diff --git a/packages/stack/src/public/effect-stack.integration.test.ts b/packages/stack/src/public/effect-stack.integration.test.ts index 36d89c0e0f..e08526d9eb 100644 --- a/packages/stack/src/public/effect-stack.integration.test.ts +++ b/packages/stack/src/public/effect-stack.integration.test.ts @@ -244,6 +244,7 @@ describe("Effect stack lifecycle handoff", () => { credentials: () => Effect.succeed(credentials), start: () => Effect.succeed(runningStatus), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -305,6 +306,7 @@ describe("Effect stack lifecycle handoff", () => { credentials: () => Effect.succeed(credentials), start: () => Effect.succeed(runningStatus), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -422,6 +424,7 @@ describe("Effect stack lifecycle handoff", () => { credentials: () => Effect.succeed(credentials), start: () => Effect.succeed(runningStatus), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: (query) => readLogs(query), }, maintenanceHandlers: { @@ -501,6 +504,7 @@ describe("Effect stack lifecycle handoff", () => { credentials: () => Effect.succeed(credentials), start: () => Effect.succeed(runningStatus), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => Effect.fail({ tag: "InvalidLogCursorError", message: "Log cursor is invalid" }), }, @@ -1370,6 +1374,7 @@ describe("Effect stack lifecycle handoff", () => { message: "Container engine command failed while starting database", }), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -1445,6 +1450,7 @@ describe("Effect stack lifecycle handoff", () => { } as const); }), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -1637,6 +1643,7 @@ describe("Effect stack lifecycle handoff", () => { return Effect.succeed(runningStatus); }, destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: emptyLogs, }; yield* startControlServer({ @@ -1698,6 +1705,7 @@ describe("Effect stack lifecycle handoff", () => { credentials: () => Effect.succeed(credentials), start: () => Effect.succeed(runningStatus), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: emptyLogs, }, maintenanceHandlers: { @@ -1740,6 +1748,7 @@ describe("Effect stack lifecycle handoff", () => { credentials: () => Effect.succeed(credentials), start: () => Effect.succeed(runningStatus), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: emptyLogs, }; const maintenanceHandlers = { @@ -2156,6 +2165,7 @@ describe("Effect stack lifecycle handoff", () => { credentials: () => invoked(credentials), start: () => invoked(runningStatus), destroy: () => invoked(undefined), + resetDatabase: () => invoked(runningStatus), logs: () => invoked({ entries: [], cursor: { opaque: "v1_0" }, running: false }), }, maintenanceHandlers: { @@ -2230,6 +2240,7 @@ describe("Effect stack lifecycle handoff", () => { start: () => Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -2289,6 +2300,7 @@ describe("Effect stack lifecycle handoff", () => { start: () => Effect.succeed(runningStatus), destroy: () => Deferred.succeed(destroyEntered, undefined).pipe(Effect.andThen(Effect.never)), + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -2352,6 +2364,7 @@ describe("Effect stack lifecycle handoff", () => { start: () => Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -2436,6 +2449,7 @@ describe("Effect stack lifecycle handoff", () => { start: () => Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { @@ -2515,6 +2529,7 @@ describe("Effect stack lifecycle handoff", () => { start: () => Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(runningStatus), logs: () => emptyLogs(), }, maintenanceHandlers: { diff --git a/packages/stack/src/public/ephemeral-postgres.integration.test.ts b/packages/stack/src/public/ephemeral-postgres.integration.test.ts new file mode 100644 index 0000000000..730eb97aa4 --- /dev/null +++ b/packages/stack/src/public/ephemeral-postgres.integration.test.ts @@ -0,0 +1,291 @@ +import { NodeServices } from "@effect/platform-node"; +import { PgClient } from "@effect/sql-pg"; +import { describe, expect, it } from "@effect/vitest"; +import { + Cause, + Duration, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Option, + Path, + Redacted, + Schedule, +} from "effect"; +import { ChildProcess } from "effect/unstable/process"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- docker availability probe for optional container cases. +import { spawnSync } from "node:child_process"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- test reserves a loopback port before fork. +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- isolated artifact cache path. +import { join } from "node:path"; +import { EphemeralPostgresError } from "./Errors.ts"; +import { createEphemeralPostgres } from "./EphemeralPostgres.ts"; +import { schemaInit } from "./SchemaInit.ts"; +import { listStacks } from "./EffectStack.ts"; +import { defaultRuntimeEnvironment, StackRuntimeEnvironment } from "../supervisor/Launcher.ts"; +import { checkHostPort } from "../supervisor/HostListener.ts"; +import type { StackRuntimePreference } from "./Runtime.ts"; + +const NATIVE_TIMEOUT_MS = 180_000; +const PASSWORD = "ephemeral-test-password"; +const JWT_SECRET = "ephemeral-test-jwt-secret-value"; + +const dockerAvailable = (): boolean => + spawnSync("docker", ["info"], { encoding: "utf8" }).status === 0; + +const artifactCacheRoot = join(tmpdir(), "supabase-stack-test-artifacts"); + +const secrets = { + databasePassword: Redacted.make(PASSWORD), + jwtSecret: Redacted.make(JWT_SECRET), +}; + +const query = (url: Redacted.Redacted, statement: string) => + Effect.scoped( + Effect.gen(function* () { + const client = yield* PgClient.PgClient; + return yield* client.unsafe(statement); + }).pipe(Effect.provide(PgClient.layer({ url, connectTimeout: "10 seconds" }))), + ); + +const withIsolatedRoot = (effect: Effect.Effect) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectory({ prefix: "supabase-eph-" }); + yield* Effect.addFinalizer(() => fs.remove(root, { recursive: true }).pipe(Effect.ignore)); + const stateRoot = path.join(root, "managed", "stacks"); + yield* fs.makeDirectory(stateRoot, { recursive: true }); + const defaults = yield* defaultRuntimeEnvironment; + return yield* effect.pipe( + Effect.provide( + Layer.succeed(StackRuntimeEnvironment, { + ...defaults, + stateRoot, + artifactCacheRoot, + }), + ), + ); + }); + +const writeForeignMarkerTar = (tarPath: string, marker: unknown) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const staging = yield* fs.makeTempDirectoryScoped(); + const data = path.join(staging, "data"); + yield* fs.makeDirectory(data); + yield* fs.writeFileString( + path.join(data, ".supabase-ephemeral-runtime"), + // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- fixture marker bytes packed into a tar. + JSON.stringify(marker), + ); + const handle = yield* ChildProcess.make("tar", ["-C", staging, "-cf", tarPath, "data"], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const code = yield* handle.exitCode; + expect(Number(code)).toBe(0); + }), + ); + +const reserveLoopbackPort = (): Effect.Effect => + Effect.callback((resume) => { + const server = createServer(); + let settled = false; + const finish = (effect: Effect.Effect) => { + if (settled) return; + settled = true; + resume(effect); + }; + server.once("error", (cause) => finish(Effect.die(cause))); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + server.close((error) => { + if (error !== undefined) { + finish(Effect.die(error)); + return; + } + finish(port > 0 ? Effect.succeed(port) : Effect.die("Unable to allocate a loopback port")); + }); + }); + return Effect.sync(() => { + if (settled) return; + settled = true; + try { + server.close(); + } catch { + // The listener never obtained a handle. + } + }); + }); + +describe.sequential("ephemeral Postgres", () => { + it.live("refuses a snapshot produced by a different runtime before starting Postgres", () => + withIsolatedRoot( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const tarPath = path.join(yield* fs.makeTempDirectoryScoped(), "foreign.tar"); + yield* writeForeignMarkerTar(tarPath, { kind: "container", engine: "docker" }); + const exit = yield* createEphemeralPostgres({ + runtime: { kind: "native" }, + restoreFrom: tarPath, + ...secrets, + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + expect(error).toBeInstanceOf(EphemeralPostgresError); + if (!(error instanceof EphemeralPostgresError)) return; + expect(error.reason).toBe("restore-mismatch"); + }), + ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.live( + "starts a native cluster, snapshots, restores, and destroys without a stack identity", + () => + withIsolatedRoot( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exportDir = yield* fs.makeTempDirectoryScoped(); + const tarPath = path.join(exportDir, "baseline.tar"); + const first = yield* createEphemeralPostgres({ runtime: { kind: "native" }, ...secrets }); + const rows = yield* query( + first.url, + "SELECT rolname FROM pg_roles WHERE rolname = 'supabase_admin'", + ); + expect(rows.length).toBeGreaterThan(0); + expect(first.runtime.kind).toBe("native"); + expect(first.artifactIdentity.startsWith("native:")).toBe(true); + const listedWhileRunning = yield* listStacks({}); + expect(listedWhileRunning.some((stack) => stack.id === first.artifactIdentity)).toBe( + false, + ); + yield* first.stop; + yield* first.exportPgData(tarPath); + const exists = yield* fs.exists(tarPath); + expect(exists).toBe(true); + + const restored = yield* createEphemeralPostgres({ + runtime: { kind: "native" }, + restoreFrom: tarPath, + ...secrets, + }); + const restoredRows = yield* query(restored.url, "SELECT current_database() AS name"); + expect(restoredRows).toEqual([{ name: "postgres" }]); + expect(restored.port).not.toBe(first.port); + + const second = yield* createEphemeralPostgres({ + runtime: { kind: "native" }, + ...secrets, + }); + expect(second.port).not.toBe(first.port); + expect(second.port).not.toBe(restored.port); + yield* query(second.url, "SELECT 1"); + }), + ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + NATIVE_TIMEOUT_MS, + ); + + it.live( + "does not leave postgres listening after an interrupted native start", + () => + withIsolatedRoot( + Effect.gen(function* () { + const port = yield* reserveLoopbackPort(); + // Fiber-owned scope so interrupt always tears the cluster down, even after start returns. + const fiber = yield* Effect.forkChild( + Effect.scoped( + createEphemeralPostgres({ runtime: { kind: "native" }, port, ...secrets }).pipe( + Effect.andThen(Effect.never), + ), + ), + ); + const url = Redacted.make( + `postgresql://${encodeURIComponent("postgres")}:${encodeURIComponent(PASSWORD)}@127.0.0.1:${port}/postgres`, + ); + yield* Effect.raceFirst( + Effect.retry(query(url, "SELECT 1"), { + schedule: Schedule.spaced("100 millis"), + }).pipe(Effect.timeout(Duration.seconds(120))), + Fiber.join(fiber), + ); + yield* Fiber.interrupt(fiber); + yield* checkHostPort("127.0.0.1", port, "database"); + }), + ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + NATIVE_TIMEOUT_MS, + ); + + it.live.skipIf(!dockerAvailable())( + "starts a container cluster, snapshots, and restores", + () => + withIsolatedRoot( + Effect.gen(function* () { + const runtime: StackRuntimePreference = { kind: "container", engine: "docker" }; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const tarPath = path.join(yield* fs.makeTempDirectoryScoped(), "baseline.tar"); + const first = yield* createEphemeralPostgres({ runtime, ...secrets }); + yield* query(first.url, "SELECT 1"); + expect(first.runtime.kind).toBe("container"); + expect(first.networkId).toEqual(expect.any(String)); + expect(first.artifactIdentity.startsWith("container:docker:")).toBe(true); + yield* first.stop; + yield* first.exportPgData(tarPath); + const restored = yield* createEphemeralPostgres({ + runtime, + restoreFrom: tarPath, + ...secrets, + }); + yield* query(restored.url, "SELECT 1"); + expect(restored.port).not.toBe(first.port); + }), + ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + NATIVE_TIMEOUT_MS, + ); + + it.live.skipIf(process.platform !== "linux" || !dockerAvailable())( + "schema-init one-shots join the cluster network and reach Postgres", + () => + withIsolatedRoot( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-linux-" }); + const runtime: StackRuntimePreference = { kind: "container", engine: "docker" }; + const cluster = yield* createEphemeralPostgres({ runtime, ...secrets }); + expect(cluster.networkId).toEqual(expect.any(String)); + yield* schemaInit(["auth"], { + kind: "ephemeral", + projectRoot, + runtime: cluster.runtime, + config: { + capabilities: { + studio: { enabled: true }, + analytics: { enabled: false }, + }, + }, + databaseUrl: Redacted.value(cluster.url), + secrets, + ...(cluster.networkId === undefined ? {} : { networkId: cluster.networkId }), + }); + const rows = yield* query( + cluster.url, + "SELECT nspname FROM pg_namespace WHERE nspname = 'auth'", + ); + expect(rows.length).toBeGreaterThan(0); + }), + ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + NATIVE_TIMEOUT_MS, + ); +}); diff --git a/packages/stack/src/public/ephemeral-postgres.unit.test.ts b/packages/stack/src/public/ephemeral-postgres.unit.test.ts new file mode 100644 index 0000000000..f8801135ef --- /dev/null +++ b/packages/stack/src/public/ephemeral-postgres.unit.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, Option } from "effect"; +import { DatabaseModule } from "../model/capabilities/database.ts"; +import { StackVersionUnsupportedError } from "./Errors.ts"; +import { resolveEphemeralPostgresRelease } from "./EphemeralPostgres.ts"; + +describe("resolveEphemeralPostgresRelease", () => { + it.effect("resolves the catalog default and a major selector", () => + Effect.gen(function* () { + const fallback = yield* resolveEphemeralPostgresRelease(); + expect(fallback.version).toBe(DatabaseModule.defaultVersion); + expect(fallback.image.length).toBeGreaterThan(0); + + const major = DatabaseModule.defaultVersion.split(".")[0]; + expect(major).toBeDefined(); + if (major === undefined) return; + const selected = yield* resolveEphemeralPostgresRelease(major); + expect(selected.version).toBe(fallback.version); + expect(selected.image).toBe(fallback.image); + }), + ); + + it.effect("fails for an unknown PostgreSQL version", () => + Effect.gen(function* () { + const exit = yield* resolveEphemeralPostgresRelease("99").pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + expect(error).toBeInstanceOf(StackVersionUnsupportedError); + }), + ); +}); diff --git a/packages/stack/src/public/index.ts b/packages/stack/src/public/index.ts index 392661da79..094e8f402f 100644 --- a/packages/stack/src/public/index.ts +++ b/packages/stack/src/public/index.ts @@ -29,3 +29,26 @@ export type { PreparedCapability, PrepareStackResult, } from "./EffectStack.ts"; +export { databaseBootstrapIdentity } from "../model/DatabaseBootstrap.ts"; +export { createEphemeralPostgres, resolveEphemeralPostgresRelease } from "./EphemeralPostgres.ts"; +export type { + CreateEphemeralPostgresOptions, + EffectEphemeralPostgres, + EphemeralPostgresRelease, + EphemeralPostgresServices, + EphemeralPostgresSettings, +} from "./EphemeralPostgres.ts"; +export { + schemaInit, + SCHEMA_INIT_CAPABILITY_NAMES, + schemaInitArtifactIdentity, +} from "./SchemaInit.ts"; +export type { + SchemaInitCapabilityName, + SchemaInitEphemeralTarget, + SchemaInitLiveTarget, + SchemaInitOptions, + SchemaInitSecrets, + SchemaInitServices, + SchemaInitTarget, +} from "./SchemaInit.ts"; diff --git a/packages/stack/src/public/promise.integration.test.ts b/packages/stack/src/public/promise.integration.test.ts index 0062a0c74e..afaa34cbef 100644 --- a/packages/stack/src/public/promise.integration.test.ts +++ b/packages/stack/src/public/promise.integration.test.ts @@ -63,6 +63,7 @@ const effectStack = (): EffectStack => start: () => Effect.succeed(status), stop: Effect.void, destroy: Effect.void, + resetDatabase: Effect.succeed(status), logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), followLogs: () => Stream.empty, }) satisfies EffectStack; diff --git a/packages/stack/src/public/reset-database.integration.test.ts b/packages/stack/src/public/reset-database.integration.test.ts new file mode 100644 index 0000000000..ae08a2b165 --- /dev/null +++ b/packages/stack/src/public/reset-database.integration.test.ts @@ -0,0 +1,115 @@ +// oxlint-disable effecttsgo/async-function -- Promise-facade live reset uses createTestStack. +// oxlint-disable-next-line effecttsgo/node-builtin-import -- docker availability probe for optional container cases. +import { execFile as execFileCallback, spawnSync } from "node:child_process"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- native storage marker path. +import { mkdir, readFile, writeFile } from "node:fs/promises"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- native storage marker path. +import { join } from "node:path"; +import { promisify } from "node:util"; +import { PgClient } from "@effect/sql-pg"; +import { Effect, Redacted } from "effect"; +import { describe, expect, it } from "vitest"; +import { createTestStack, type TestStack } from "../testing.ts"; +import type { StackRuntimePreference } from "./Runtime.ts"; + +const RESET_TIMEOUT_MS = 180_000; +const execFile = promisify(execFileCallback); +const MARKER_TABLE = "public.stack_reset_marker"; + +const dockerAvailable = (): boolean => + spawnSync("docker", ["info"], { encoding: "utf8" }).status === 0; + +const query = async (url: string, statement: string): Promise> => + Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const client = yield* PgClient.PgClient; + return yield* client.unsafe(statement); + }).pipe( + Effect.provide(PgClient.layer({ url: Redacted.make(url), connectTimeout: "10 seconds" })), + ), + ), + ); + +const volumeWorkloadIds = async (stackId: string): Promise> => { + const listed = await execFile("docker", [ + "volume", + "ls", + "-q", + "--filter", + `label=com.supabase.stack.stackId=${stackId}`, + ]); + const ids = listed.stdout + .trim() + .split("\n") + .filter((value) => value.length > 0); + if (ids.length === 0) return []; + const inspected = await execFile("docker", [ + "inspect", + "--format", + '{{index .Labels "com.supabase.stack.workloadId"}}', + ...ids, + ]); + return inspected.stdout + .trim() + .split("\n") + .filter((value) => value.length > 0); +}; + +const resetAndAssert = async (stack: TestStack, runtime: StackRuntimePreference): Promise => { + const before = await stack.status(); + const credentials = await stack.credentials(); + await query(credentials.database.url, `CREATE TABLE ${MARKER_TABLE} (id integer PRIMARY KEY)`); + const storageMarker = join(stack.stateRoot, stack.id, "data", "storage", "keep.txt"); + if (runtime.kind === "native") { + await mkdir(join(stack.stateRoot, stack.id, "data", "storage"), { recursive: true }); + await writeFile(storageMarker, "keep"); + } + const volumesBefore = runtime.kind === "container" ? await volumeWorkloadIds(stack.id) : []; + + const after = await stack.resetDatabase(); + expect(after.id).toBe(stack.id); + expect(after.endpoints).toEqual(before.endpoints); + expect(after.lifecycle).toBe("running"); + const database = after.capabilities.find((capability) => capability.name === "database"); + expect(database?.state).toBe("ready"); + + const leftover = await query( + (await stack.credentials()).database.url, + `SELECT to_regclass('${MARKER_TABLE}') AS name`, + ); + expect(leftover).toEqual([{ name: null }]); + + if (runtime.kind === "native") { + expect(await readFile(storageMarker, "utf8")).toBe("keep"); + } else { + const volumesAfter = await volumeWorkloadIds(stack.id); + expect(volumesAfter.filter((id) => id !== "database:database")).toEqual( + volumesBefore.filter((id) => id !== "database:database"), + ); + } +}; + +describe("resetDatabase", () => { + it( + "wipes native Postgres while keeping identity, ports, and storage data", + async () => { + await using stack = await createTestStack({ + runtime: { kind: "native" }, + }); + await resetAndAssert(stack, { kind: "native" }); + }, + RESET_TIMEOUT_MS, + ); + + it.skipIf(!dockerAvailable())( + "wipes container Postgres while keeping identity, ports, and non-database volumes", + async () => { + await using stack = await createTestStack({ + runtime: { kind: "container", engine: "docker" }, + }); + await resetAndAssert(stack, { kind: "container", engine: "docker" }); + }, + RESET_TIMEOUT_MS, + ); +}); diff --git a/packages/stack/src/public/testing.integration.test.ts b/packages/stack/src/public/testing.integration.test.ts index dbec010f1b..b51a20db4a 100644 --- a/packages/stack/src/public/testing.integration.test.ts +++ b/packages/stack/src/public/testing.integration.test.ts @@ -121,6 +121,15 @@ const fakeStack = (events: Array, options: FakeStackOptions = {}): Promi events.push("destroy"); if (failStart) throw new Error("destroy failed"); }), + resetDatabase: () => + fixturePromise(() => + status( + reachesReadiness ? "running" : "stopped", + includeApi, + functionsState, + failedCapability, + ), + ), logs: () => fixturePromise(() => ({ entries: [], cursor: { opaque: "v1_0" }, running: false })), followLogs: () => stream([]), }; diff --git a/packages/stack/src/public/whole-stack.e2e.test.ts b/packages/stack/src/public/whole-stack.e2e.test.ts index 33141e1f82..966a62b63a 100644 --- a/packages/stack/src/public/whole-stack.e2e.test.ts +++ b/packages/stack/src/public/whole-stack.e2e.test.ts @@ -598,16 +598,24 @@ const databaseQuery = async ( } }; +const requireApi = (credentials: PromiseStackCredentials) => { + if (credentials.api === undefined) throw new Error("API credentials are unavailable"); + return credentials.api; +}; + const apiHeaders = ( credentials: PromiseStackCredentials, - token: string = credentials.api.anonJwt, -): Record => ({ - apikey: credentials.api.publishableKey, - Authorization: `Bearer ${token}`, -}); + token?: string, +): Record => { + const api = requireApi(credentials); + return { + apikey: api.publishableKey, + Authorization: `Bearer ${token ?? api.anonJwt}`, + }; +}; const serviceHeaders = (credentials: PromiseStackCredentials): Record => - apiHeaders(credentials, credentials.api.serviceRoleJwt); + apiHeaders(credentials, requireApi(credentials).serviceRoleJwt); const functionSource = (table: string, marker: string): string => ` Deno.serve(async () => { @@ -925,7 +933,9 @@ const exerciseWholeStackRealtime = async ( const socket = await (async (): Promise => { try { return await activate(stack, "realtime", async () => { - const candidate = await openSocket(makeRealtimeUrl(api, credentials.api.publishableKey)); + const candidate = await openSocket( + makeRealtimeUrl(api, requireApi(credentials).publishableKey), + ); openedSocket = candidate; return candidate; }); @@ -1132,7 +1142,7 @@ const reactivateWholeStackCapabilities = async ( await request(api.url, "/auth/v1/settings", { headers: apiHeaders(credentials) }); }); await activate(stack, "realtime", async () => { - const probe = await openSocket(makeRealtimeUrl(api, credentials.api.publishableKey)); + const probe = await openSocket(makeRealtimeUrl(api, requireApi(credentials).publishableKey)); probe.close(); }); await activate(stack, "storage", async () => { @@ -1510,12 +1520,13 @@ describe("managed Supabase stack whole-stack E2E", () => { name: `stack-cli-consumer-${identity}`, runtime: mode.runtime, }); - await ordinary.start(); + const ordinaryStack = ordinary; + await ordinaryStack.start(); helper = await createTestStack({ name: `stack-helper-consumer-${identity}`, runtime: mode.runtime, }); - const ordinaryStatus = await ordinary.status(); + const ordinaryStatus = await ordinaryStack.status(); const helperStatus = await helper.status(); expect(ordinaryStatus.lifecycle).toBe("running"); expect(helperStatus.lifecycle).toBe("running"); @@ -1524,7 +1535,7 @@ describe("managed Supabase stack whole-stack E2E", () => { ); expect(endpoint(ordinaryStatus, "api").port).not.toBe(endpoint(helperStatus, "api").port); const scoped = await listStacks({ projectRoot: ordinaryRoot }); - expect(scoped.map(({ id }) => id)).toContain(ordinary.id); + expect(scoped.map(({ id }) => id)).toContain(ordinaryStack.id); expect(scoped.map(({ id }) => id)).not.toContain(helper.id); const all = await listStacks(); expect(all.map(({ id }) => id)).toEqual(expect.arrayContaining([ordinary.id, helper.id])); diff --git a/packages/stack/src/runtime/ContainerEngine.ts b/packages/stack/src/runtime/ContainerEngine.ts index 9162e706af..4d68544251 100644 --- a/packages/stack/src/runtime/ContainerEngine.ts +++ b/packages/stack/src/runtime/ContainerEngine.ts @@ -149,6 +149,8 @@ export interface ContainerContainerSpec { /** Path to an owned 0600 env file. Secret values must never be argv. */ readonly envFile?: string; readonly networkAliases?: ReadonlyArray; + /** Linux Engine needs `host.docker.internal:host-gateway` to reach published loopback. */ + readonly extraHosts?: ReadonlyArray; } export type ContainerCommand = @@ -224,6 +226,7 @@ export const serializeCommonContainerCommand = ( command.spec.networkAliases === undefined ? [] : command.spec.networkAliases.flatMap((alias) => ["--network-alias", alias]); + const extraHosts = (command.spec.extraHosts ?? []).flatMap((host) => ["--add-host", host]); const entrypoint = command.spec.entrypoint === undefined ? [] : ["--entrypoint", command.spec.entrypoint]; return { @@ -234,6 +237,7 @@ export const serializeCommonContainerCommand = ( "--network", command.spec.network, ...networkAliases, + ...extraHosts, ...containerLabels(command.spec.labels), ...bindMounts, ...volumeMounts, @@ -732,7 +736,10 @@ export const makeContainerEngineCore = (options: ContainerEngineOptions): Contai : Effect.fail( new ContainerCommandError({ operation, - message: `Container engine command failed (${result.exitCode})`, + message: + result.stderr.trim().length > 0 + ? `Container engine command failed (${result.exitCode}): ${result.stderr.trim()}` + : `Container engine command failed (${result.exitCode})`, }), ), ), diff --git a/packages/stack/src/runtime/ContainerEngineResolver.ts b/packages/stack/src/runtime/ContainerEngineResolver.ts index 8c0f3d8812..d42570351a 100644 --- a/packages/stack/src/runtime/ContainerEngineResolver.ts +++ b/packages/stack/src/runtime/ContainerEngineResolver.ts @@ -1,5 +1,7 @@ import { Context, Effect } from "effect"; import type { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; +import { ContainerEngineError } from "../public/Errors.ts"; +import type { StackRuntime } from "../public/Runtime.ts"; import { makeProcessCommandRunner, type ContainerEngine, @@ -65,3 +67,21 @@ export const resolveContainerEngine = ( resolver?: ContainerEngineResolverShape, ): Effect.Effect => (resolver ?? defaultContainerEngineResolver).resolve(kind); + +/** Docker when the client is installed, otherwise native. */ +export const selectDefaultRuntime = ( + resolver?: ContainerEngineResolverShape, +): Effect.Effect => + (resolver ?? defaultContainerEngineResolver).isInstalled("docker").pipe( + Effect.map((installed): StackRuntime => + installed ? { kind: "container", engine: "docker" } : { kind: "native" }, + ), + Effect.mapError( + (error) => + new ContainerEngineError({ + engine: "docker", + message: `Unable to determine whether Docker is installed: ${error.message}`, + cause: error, + }), + ), + ); diff --git a/packages/stack/src/runtime/ContainerEngineResolver.unit.test.ts b/packages/stack/src/runtime/ContainerEngineResolver.unit.test.ts new file mode 100644 index 0000000000..dc0208eabe --- /dev/null +++ b/packages/stack/src/runtime/ContainerEngineResolver.unit.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { + selectDefaultRuntime, + type ContainerEngineResolverShape, +} from "./ContainerEngineResolver.ts"; + +const unusedSpawner = ChildProcessSpawner.make(() => Effect.die("unused")); + +const resolver = (installed: boolean): ContainerEngineResolverShape => ({ + isInstalled: () => Effect.succeed(installed), + resolve: () => Effect.die("unused"), +}); + +describe("selectDefaultRuntime", () => { + it.effect("selects Docker when the client is installed", () => + Effect.gen(function* () { + expect(yield* selectDefaultRuntime(resolver(true))).toEqual({ + kind: "container", + engine: "docker", + }); + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, unusedSpawner)), + ); + + it.effect("selects native when Docker is not installed", () => + Effect.gen(function* () { + expect(yield* selectDefaultRuntime(resolver(false))).toEqual({ kind: "native" }); + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, unusedSpawner)), + ); +}); diff --git a/packages/stack/src/runtime/ContainerRuntime.ts b/packages/stack/src/runtime/ContainerRuntime.ts index d8c506c19a..c108756589 100644 --- a/packages/stack/src/runtime/ContainerRuntime.ts +++ b/packages/stack/src/runtime/ContainerRuntime.ts @@ -124,6 +124,10 @@ const nameFor = (key: RuntimeWorkloadKey, role: ContainerResourceRole): string = ? `supabase-${key.stackId.slice(0, 16)}-network` : `supabase-${key.stackId.slice(0, 16)}-${key.workloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-${role}`; +/** Distinct from {@link nameFor}(..., "workload") so one-shots cannot collide with an eager main container. */ +export const schemaInitContainerName = (key: RuntimeWorkloadKey): string => + `supabase-${key.stackId.slice(0, 16)}-${key.workloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-schema-init`; + const networkLabelsFor = ( key: RuntimeWorkloadKey, ownerSessionId: string, @@ -222,6 +226,107 @@ const toContainerEngineError = ( cause: error, }); +const withEngine = ( + engine: ContainerEngine, + key: Pick, + effect: Effect.Effect, +): Effect.Effect => + effect.pipe( + Effect.mapError((error) => toDriverError(key, toContainerEngineError(engine.kind, error))), + ); + +/** Runs one service-owned one-shot container: create, start, wait for exit 0, remove. */ +export const runContainerStartupProcess = (input: { + readonly engine: ContainerEngine; + readonly key: RuntimeWorkloadKey; + readonly specification: ContainerContainerSpec; + readonly timeout: Duration.Input; + readonly logStore?: LogStore; + readonly capability?: PlannedWorkload["capability"]; + readonly runtimeScope?: Scope.Scope; +}): Effect.Effect => { + let logFiber: Fiber.Fiber | undefined; + const acquire = withEngine( + input.engine, + input.key, + input.engine.createContainer(input.specification), + ); + const use = (container: ContainerResource): Effect.Effect => + Effect.gen(function* () { + yield* withEngine(input.engine, input.key, input.engine.startContainer(container.id)); + const logStore = input.logStore; + const runtimeScope = input.runtimeScope; + const capability = input.capability; + if (logStore !== undefined && runtimeScope !== undefined && capability !== undefined) { + const consume = input.engine.streamLogs(container.id, { tail: "all" }).pipe( + Stream.runForEach((line) => + logStore + .append({ + source: capability, + stream: line.stream, + message: line.message, + }) + .pipe(Effect.asVoid), + ), + Effect.mapError((error) => + toDriverError(input.key, toContainerEngineError(input.engine.kind, error)), + ), + ); + logFiber = yield* Effect.forkIn(consume, runtimeScope); + } + const exitCode = yield* withEngine( + input.engine, + input.key, + input.engine.waitContainer(container.id), + ); + const logs = logFiber === undefined ? Exit.succeed(undefined) : yield* Fiber.await(logFiber); + const logFailure = + Exit.isFailure(logs) && !Cause.hasInterruptsOnly(logs.cause) ? logs.cause : undefined; + const exitFailure = + exitCode === 0 + ? undefined + : toDriverError( + input.key, + new Error( + `Container startup process exited with code ${String(exitCode)} for ${input.key.workloadId}`, + ), + ); + if (exitFailure !== undefined) { + const exitCause = Cause.fail(exitFailure); + return yield* logFailure !== undefined + ? Effect.failCause(Cause.combine(exitCause, logFailure)) + : Effect.failCause(exitCause); + } + if (logFailure !== undefined) return yield* Effect.failCause(logFailure); + }); + const release = ( + container: ContainerResource, + useExit: Exit.Exit, + ): Effect.Effect => + Effect.gen(function* () { + if (logFiber !== undefined) yield* Fiber.interrupt(logFiber); + const removed = yield* Effect.exit( + withEngine(input.engine, input.key, input.engine.removeContainer(container.id)), + ); + if (Exit.isFailure(removed)) + return yield* Effect.failCause( + Exit.isFailure(useExit) ? Cause.combine(useExit.cause, removed.cause) : removed.cause, + ); + }); + return Effect.acquireUseRelease(acquire, use, release).pipe( + Effect.timeoutOrElse({ + duration: input.timeout, + orElse: () => + Effect.fail( + toDriverError( + input.key, + new Error(`Container startup process timed out for ${input.key.workloadId}`), + ), + ), + }), + ); +}; + const containerArtifact = (workload: PlannedWorkload): ContainerArtifact | undefined => workload.selected.kind === "container" ? workload.selected : undefined; @@ -445,94 +550,35 @@ export const makeContainerRuntime = ( }>, ): Effect.Effect => { const labels = startupLabelsFor(key, options.ownerSessionId); - const specification: ContainerContainerSpec = { - // Reuse the main name so crash-orphaned init containers are exact collisions to clean up. - name: nameFor(key, "workload"), - image: context.artifact.image, - labels, - network: context.network.id, - mounts: context.resolution.mounts ?? [], - volumeMounts: - context.volumeRequest === undefined ? [] : [volumeMountFor(key, context.volumeRequest)], - publications: [], - role: "workload", - entrypoint: startupProcess.entrypoint, - command: startupProcess.command, - ...(context.resolution.envFile === undefined + return runContainerStartupProcess({ + engine: options.engine, + key, + timeout: startupProcessTimeout, + specification: { + // Reuse the main name so crash-orphaned init containers are exact collisions to clean up. + name: nameFor(key, "workload"), + image: context.artifact.image, + labels, + network: context.network.id, + mounts: context.resolution.mounts ?? [], + volumeMounts: + context.volumeRequest === undefined ? [] : [volumeMountFor(key, context.volumeRequest)], + publications: [], + role: "workload", + entrypoint: startupProcess.entrypoint, + command: startupProcess.command, + ...(context.resolution.envFile === undefined + ? {} + : { envFile: context.resolution.envFile }), + }, + ...(options.logStore === undefined ? {} - : { envFile: context.resolution.envFile }), - }; - let logFiber: Fiber.Fiber | undefined; - const acquire = withEngine(key, options.engine.createContainer(specification)); - const logStore = options.logStore; - const use = (container: ContainerResource): Effect.Effect => - Effect.gen(function* () { - yield* withEngine(key, options.engine.startContainer(container.id)); - if (logStore !== undefined) { - const consume = options.engine.streamLogs(container.id, { tail: "all" }).pipe( - Stream.runForEach((line) => - logStore - .append({ - source: workload.capability, - stream: line.stream, - message: line.message, - }) - .pipe(Effect.asVoid), - ), - Effect.mapError((error) => - toDriverError(key, toContainerEngineError(options.engine.kind, error)), - ), - ); - logFiber = yield* Effect.forkIn(consume, runtimeScope); - } - const exitCode = yield* withEngine(key, options.engine.waitContainer(container.id)); - const logs = - logFiber === undefined ? Exit.succeed(undefined) : yield* Fiber.await(logFiber); - const logFailure = - Exit.isFailure(logs) && !Cause.hasInterruptsOnly(logs.cause) ? logs.cause : undefined; - const exitFailure = - exitCode === 0 - ? undefined - : toDriverError( - key, - new Error( - `Container startup process exited with code ${String(exitCode)} for ${key.workloadId}`, - ), - ); - if (exitFailure !== undefined) { - const exitCause = Cause.fail(exitFailure); - return yield* logFailure !== undefined - ? Effect.failCause(Cause.combine(exitCause, logFailure)) - : Effect.failCause(exitCause); - } - if (logFailure !== undefined) return yield* Effect.failCause(logFailure); - }); - const release = ( - container: ContainerResource, - useExit: Exit.Exit, - ): Effect.Effect => - Effect.gen(function* () { - if (logFiber !== undefined) yield* Fiber.interrupt(logFiber); - const removed = yield* Effect.exit( - withEngine(key, options.engine.removeContainer(container.id)), - ); - if (Exit.isFailure(removed)) - return yield* Effect.failCause( - Exit.isFailure(useExit) ? Cause.combine(useExit.cause, removed.cause) : removed.cause, - ); - }); - return Effect.acquireUseRelease(acquire, use, release).pipe( - Effect.timeoutOrElse({ - duration: startupProcessTimeout, - orElse: () => - Effect.fail( - toDriverError( - key, - new Error(`Container startup process timed out for ${key.workloadId}`), - ), - ), - }), - ); + : { + logStore: options.logStore, + capability: workload.capability, + runtimeScope, + }), + }); }; const start = ( @@ -1038,11 +1084,27 @@ export const makeContainerRuntime = ( }), ); + const wipePersistentData = (key: RuntimeWorkloadKey): Effect.Effect => + registration.withPermit( + Effect.gen(function* () { + const entries = yield* withEngine(key, options.engine.listResources(key.stackId)); + const volumes = entries.filter( + (entry) => + entry.kind === "volume" && + entry.labels.role === "volume" && + entry.labels.workloadId === key.workloadId, + ); + for (const volume of volumes) + yield* withEngine(key, options.engine.removeVolume(volume.id)); + }), + ); + return { observe, start, stop, remove, cleanup, + wipePersistentData, } satisfies RuntimeDriver; }); diff --git a/packages/stack/src/runtime/EphemeralPostgres.ts b/packages/stack/src/runtime/EphemeralPostgres.ts new file mode 100644 index 0000000000..dbf9f13c03 --- /dev/null +++ b/packages/stack/src/runtime/EphemeralPostgres.ts @@ -0,0 +1,1088 @@ +import { PgClient } from "@effect/sql-pg"; +import { + Crypto, + Duration, + Effect, + Exit, + FileSystem, + Option, + Path, + Redacted, + Schedule, + Schema, + Scope, + Semaphore, +} from "effect"; +import { ChildProcess } from "effect/unstable/process"; +import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- loopback bind is the port allocator. +import { createServer } from "node:net"; +import { DatabaseBootstrapError } from "../model/DatabaseBootstrap.ts"; +import { + DEFAULT_DATABASE_HEALTH_TIMEOUT, + parseGoDuration, +} from "../model/capabilities/database.ts"; +import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; +import { + ContainerEngineError, + EphemeralPostgresError, + PortUnavailableError, + StackPreparationError, + type EphemeralPostgresCreateError, +} from "../public/Errors.ts"; +import { + resolveEphemeralPostgresRelease, + type CreateEphemeralPostgresOptions, + type EffectEphemeralPostgres, +} from "../public/EphemeralPostgres.ts"; +import type { StackRuntime } from "../public/Runtime.ts"; +import { StackIdSchema, type StackId } from "../public/StackId.ts"; +import { defaultRuntimeEnvironment, StackRuntimeEnvironment } from "../supervisor/Launcher.ts"; +import { checkHostPort } from "../supervisor/HostListener.ts"; +import { probeReadiness } from "./ReadinessProbe.ts"; +import { + defaultNativeProcessLauncher, + spawnNativeProcess, + type NativeProcess, +} from "./NativeProcess.ts"; +import { bootstrapManagedPostgres } from "./PostgresDatabaseSession.ts"; +import { makeProductionRuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; +import { + resolveContainerEngine, + ContainerEngineResolver, + selectDefaultRuntime, + type ContainerEngineResolverShape, +} from "./ContainerEngineResolver.ts"; +import type { ContainerEngine } from "./ContainerEngine.ts"; +import { encodeRuntimeEnvFile } from "./RuntimeEnvFile.ts"; +import { containerAliasFor } from "../model/WorkloadCatalog.ts"; + +const DATABASE_WORKLOAD_ID = "database:database"; +const PGDATA_DIR_NAME = "data"; +const CONTAINER_PGDATA_PARENT = "/var/lib/postgresql"; +const SNAPSHOT_MOUNT = "/snapshot"; +const BUSYBOX = "/usr/bin/busybox"; +const RUNTIME_MARKER = ".supabase-ephemeral-runtime"; +const DEFAULT_JWT_EXPIRY = 3600; + +const RuntimeMarkerSchema = Schema.Struct({ + kind: Schema.Literals(["native", "container"] as const), + engine: Schema.optionalKey(Schema.Literals(["docker", "podman"] as const)), +}); +type RuntimeMarker = Schema.Schema.Type; + +const ephemeralError = ( + message: string, + fields: Omit[0], "message"> = {}, +) => new EphemeralPostgresError({ message, ...fields }); + +const resolvedRuntime = ( + preference: CreateEphemeralPostgresOptions["runtime"] | undefined, + resolver: ContainerEngineResolverShape | undefined, +): Effect.Effect => { + if (preference !== undefined) { + return Effect.succeed( + preference.kind === "container" + ? { kind: "container", engine: preference.engine ?? "docker" } + : { kind: "native" }, + ); + } + return selectDefaultRuntime(resolver); +}; + +const plannedWorkload = ( + version: string, + image: string, + runtime: StackRuntime, +): PlannedWorkload => ({ + id: DATABASE_WORKLOAD_ID, + capability: "database", + bootstrap: "database", + dependencies: [], + readiness: { portField: "database" }, + artifacts: { + native: { kind: "native", release: version }, + container: { kind: "container", image }, + }, + selected: + runtime.kind === "native" ? { kind: "native", release: version } : { kind: "container", image }, +}); + +const postgresArgs = ( + port: number, + runtime: StackRuntime, + settings: CreateEphemeralPostgresOptions["postgresSettings"], +): ReadonlyArray => { + const tuned = Object.entries(settings ?? {}).flatMap(([key, value]) => { + if (value === undefined) return []; + const rendered = String(value); + return rendered.length === 0 ? [] : ["-c", `${key}=${rendered}`]; + }); + return [ + "-p", + String(port), + "-c", + runtime.kind === "container" ? "listen_addresses=*" : "listen_addresses=127.0.0.1", + ...tuned, + ]; +}; + +const postgresEnv = (input: { + readonly port: number; + readonly dataPath: string; + readonly password: string; +}): Record => ({ + SUPABASE_STACK_WORKLOAD: DATABASE_WORKLOAD_ID, + SUPABASE_STACK_PRIVATE_PORT: String(input.port), + PGDATA: input.dataPath, + POSTGRES_USER: "supabase_admin", + POSTGRES_DB: "postgres", + POSTGRES_PASSWORD: input.password, + TZDIR: "/var/db/timezone/zoneinfo", +}); + +const databaseUrl = (port: number, password: string): string => + `postgresql://${encodeURIComponent("postgres")}:${encodeURIComponent(password)}@127.0.0.1:${port}/postgres`; + +const markerFor = (runtime: StackRuntime): RuntimeMarker => + runtime.kind === "native" ? { kind: "native" } : { kind: "container", engine: runtime.engine }; + +const encodeMarker = (marker: RuntimeMarker): string => JSON.stringify(marker); + +const decodeMarker = (text: string): Effect.Effect => + Schema.decodeEffect(Schema.fromJsonString(RuntimeMarkerSchema))(text).pipe( + Effect.mapError(() => + ephemeralError("Ephemeral Postgres snapshot marker is invalid", { reason: "snapshot" }), + ), + ); + +const sameRuntime = (left: RuntimeMarker, right: StackRuntime): boolean => + left.kind === right.kind && (right.kind === "native" || left.engine === right.engine); + +const allocateLoopbackPort = ( + requested: number | undefined, +): Effect.Effect => { + if (requested !== undefined) + return checkHostPort("127.0.0.1", requested, "database").pipe(Effect.as(requested)); + return Effect.callback((resume) => { + const server = createServer(); + let settled = false; + const finish = (effect: Effect.Effect) => { + if (settled) return; + settled = true; + resume(effect); + }; + server.once("error", (cause) => + finish(Effect.fail(ephemeralError("Unable to allocate a loopback port", { cause }))), + ); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + server.close(() => + finish( + port > 0 + ? Effect.succeed(port) + : Effect.fail(ephemeralError("Unable to allocate a loopback port")), + ), + ); + }); + return Effect.sync(() => { + if (settled) return; + settled = true; + try { + server.close(); + } catch { + // The listener never obtained a handle. + } + }); + }); +}; + +const runTar = ( + args: ReadonlyArray, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const handle = yield* ChildProcess.make("tar", args, { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }).pipe(Effect.mapError((cause) => ephemeralError("Unable to start tar", { cause }))); + const code = yield* handle.exitCode.pipe( + Effect.mapError((cause) => ephemeralError("tar failed", { cause })), + ); + if (Number(code) !== 0) + return yield* ephemeralError(`tar failed (${String(code)})`, { reason: "snapshot" }); + }), + ); + +const writeEnvFile = ( + filePath: string, + values: Readonly>, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const text = yield* encodeRuntimeEnvFile(values).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to write Postgres environment file", { cause, path: filePath }), + ), + ); + yield* fs + .writeFileString(filePath, text) + .pipe( + Effect.mapError((cause) => + ephemeralError("Unable to write Postgres environment file", { cause, path: filePath }), + ), + ); + yield* fs.chmod(filePath, 0o600).pipe(Effect.ignore); + return filePath; + }); + +const waitForPostgres = ( + port: number, + healthTimeout: string, +): Effect.Effect => + Effect.try({ + try: () => parseGoDuration(healthTimeout), + catch: (cause) => ephemeralError("Invalid database health timeout", { cause }), + }).pipe( + Effect.flatMap((deadline) => + probeReadiness( + { mode: "tcp", host: "127.0.0.1", port }, + { deadline: Duration.isZero(deadline) ? Duration.seconds(1) : deadline }, + ).pipe( + Effect.mapError((cause) => + ephemeralError("Ephemeral Postgres did not become ready", { cause }), + ), + ), + ), + ); + +const pingAdvertised = ( + port: number, + password: string, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const client = yield* PgClient.PgClient; + yield* client.unsafe("SELECT 1"); + }).pipe( + Effect.provide( + PgClient.layer({ + url: Redacted.make(databaseUrl(port, password)), + connectTimeout: "2 seconds", + }), + ), + ), + ).pipe( + Effect.mapError((cause) => + ephemeralError("Ephemeral Postgres did not accept a connection", { cause }), + ), + ); + +const waitForAdvertised = ( + port: number, + password: string, + healthTimeout: string, +): Effect.Effect => + Effect.try({ + try: () => parseGoDuration(healthTimeout), + catch: (cause) => ephemeralError("Invalid database health timeout", { cause }), + }).pipe( + Effect.flatMap((deadline) => + Effect.timeout( + Effect.retry(pingAdvertised(port, password), { + schedule: Schedule.spaced("100 millis"), + }), + Duration.isZero(deadline) ? Duration.seconds(1) : deadline, + ).pipe( + Effect.mapError((cause) => + ephemeralError("Ephemeral Postgres did not become ready", { cause }), + ), + ), + ), + ); + +const bootstrap = ( + port: number, + options: CreateEphemeralPostgresOptions, + healthTimeout: string, +): Effect.Effect => + Effect.try({ + try: () => parseGoDuration(healthTimeout), + catch: (cause) => ephemeralError("Invalid database health timeout", { cause }), + }).pipe( + Effect.flatMap((deadline) => + Effect.timeout( + Effect.retry( + bootstrapManagedPostgres({ + host: "127.0.0.1", + port, + databasePassword: options.databasePassword, + jwtSecret: options.jwtSecret, + jwtExpiry: options.jwtExpiry ?? DEFAULT_JWT_EXPIRY, + }), + { + schedule: Schedule.spaced("100 millis"), + while: (error) => error instanceof DatabaseBootstrapError && error.retryable === true, + }, + ), + Duration.isZero(deadline) ? Duration.seconds(1) : deadline, + ).pipe( + Effect.mapError((cause) => + ephemeralError("Ephemeral Postgres bootstrap failed", { reason: "bootstrap", cause }), + ), + ), + ), + ); + +interface NativeResources { + readonly kind: "native"; + process?: NativeProcess; + processScope?: Scope.Closeable; +} + +interface ContainerResources { + readonly kind: "container"; + readonly engine: ContainerEngine; + networkId?: string; + volumeId?: string; + containerId?: string; +} + +type RuntimeResources = NativeResources | ContainerResources; + +interface Cluster { + readonly identity: StackId; + readonly root: string; + readonly dataPath: string; + readonly host: "127.0.0.1"; + readonly port: number; + readonly version: string; + readonly runtime: StackRuntime; + readonly artifactIdentity: string; + readonly executable?: string; + readonly image?: string; + readonly lifecycle: Semaphore.Semaphore; + running: boolean; + bootstrapped: boolean; + resources: RuntimeResources; +} + +const resourceName = (identity: StackId, role: string): string => + `supabase-eph-${identity.slice(0, 16)}-${role}`; + +const createIdentity = (crypto: Crypto.Crypto): Effect.Effect => + Effect.gen(function* () { + const first = yield* crypto.randomUUIDv4; + const second = yield* crypto.randomUUIDv4; + return yield* Schema.decodeEffect(StackIdSchema)(`${first}${second}`.replaceAll("-", "")); + }).pipe( + Effect.mapError((cause) => ephemeralError("Unable to allocate ephemeral identity", { cause })), + ); + +const writeRuntimeMarker = ( + cluster: Cluster, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const markerPath = `${cluster.dataPath}/${RUNTIME_MARKER}`; + const encoded = encodeMarker(markerFor(cluster.runtime)); + if (cluster.resources.kind === "native") { + yield* fs.writeFileString(markerPath, encoded).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to write snapshot runtime marker", { + cause, + path: markerPath, + reason: "snapshot", + }), + ), + ); + return; + } + const containerId = cluster.resources.containerId; + if (containerId === undefined) + return yield* ephemeralError("Ephemeral Postgres container is missing", { + reason: "snapshot", + }); + const tempPath = `${cluster.root}/${RUNTIME_MARKER}`; + yield* fs.writeFileString(tempPath, encoded).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to write snapshot runtime marker", { + cause, + path: tempPath, + reason: "snapshot", + }), + ), + ); + yield* cluster.resources.engine + .copyToContainer( + containerId, + tempPath, + `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}/${RUNTIME_MARKER}`, + ) + .pipe( + Effect.mapError((cause) => + ephemeralError("Unable to copy snapshot runtime marker", { cause, reason: "snapshot" }), + ), + ); + }); + +const snapshotFileName = ( + tarPath: string, + path: Path.Path, +): Effect.Effect => { + const name = path.basename(tarPath); + if (name.length === 0 || name === "." || name === "..") + return Effect.fail( + ephemeralError("Ephemeral Postgres snapshot path is invalid", { + reason: "snapshot", + path: tarPath, + }), + ); + return Effect.succeed(name); +}; + +/** Catalog image has no tar on PATH; busybox tar archives the volume in place. */ +const runVolumeTar = ( + cluster: Cluster, + tarPath: string, + mode: "create" | "extract", +): Effect.Effect => + Effect.gen(function* () { + if (cluster.resources.kind !== "container") return; + const { engine, networkId, volumeId } = cluster.resources; + if (networkId === undefined || volumeId === undefined) + return yield* ephemeralError("Ephemeral Postgres volume is unavailable", { + reason: "snapshot", + }); + const image = cluster.image; + if (image === undefined) + return yield* ephemeralError("Ephemeral Postgres image is unavailable", { + reason: "snapshot", + }); + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + const fileName = yield* snapshotFileName(tarPath, path); + const parent = path.dirname(tarPath); + yield* fs.makeDirectory(parent, { recursive: true }).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to create snapshot directory", { + cause, + path: parent, + reason: "snapshot", + }), + ), + ); + const snapshotPath = `${SNAPSHOT_MOUNT}/${fileName}`; + const command = + mode === "create" + ? ["tar", "-C", CONTAINER_PGDATA_PARENT, "-cf", snapshotPath, PGDATA_DIR_NAME] + : ["tar", "-C", CONTAINER_PGDATA_PARENT, "-xf", snapshotPath]; + yield* Effect.acquireUseRelease( + engine + .createContainer({ + name: resourceName(cluster.identity, "snapshot"), + image, + labels: { + stackId: cluster.identity, + ownerSessionId: cluster.identity.slice(0, 32), + workloadId: `${DATABASE_WORKLOAD_ID}:snapshot`, + role: "workload", + }, + network: networkId, + mounts: [{ source: parent, target: SNAPSHOT_MOUNT, readOnly: mode === "extract" }], + volumeMounts: [ + { + volume: volumeId, + target: `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}`, + readOnly: false, + }, + ], + publications: [], + role: "workload", + entrypoint: BUSYBOX, + command, + }) + .pipe( + Effect.mapError((cause) => + ephemeralError("Unable to create snapshot helper", { cause, reason: "snapshot" }), + ), + ), + (created) => + Effect.gen(function* () { + yield* engine.startContainer(created.id).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to start snapshot helper", { + cause, + reason: "snapshot", + }), + ), + ); + const code = yield* engine.waitContainer(created.id).pipe( + Effect.mapError((cause) => + ephemeralError("Snapshot helper did not finish", { + cause, + reason: "snapshot", + }), + ), + ); + if (code !== 0) + return yield* ephemeralError(`Snapshot helper failed (${String(code)})`, { + reason: "snapshot", + }); + }), + (created) => engine.removeContainer(created.id).pipe(Effect.ignore), + ); + }); + +const verifyRestoredMarker = ( + cluster: Cluster, + restoreFrom: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const markerPath = `${cluster.dataPath}/${RUNTIME_MARKER}`; + const exists = yield* fs.exists(markerPath).pipe(Effect.orElseSucceed(() => false)); + if (!exists) + return yield* ephemeralError("Ephemeral Postgres snapshot is missing a runtime marker", { + reason: "restore-mismatch", + path: restoreFrom, + }); + const marker = yield* fs.readFileString(markerPath).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to read snapshot runtime marker", { + cause, + path: markerPath, + reason: "snapshot", + }), + ), + Effect.flatMap(decodeMarker), + ); + if (!sameRuntime(marker, cluster.runtime)) + return yield* ephemeralError( + "Ephemeral Postgres snapshot was produced by a different runtime", + { + reason: "restore-mismatch", + path: restoreFrom, + }, + ); + }); + +const stopNative = (cluster: Cluster): Effect.Effect => + Effect.gen(function* () { + if (cluster.resources.kind !== "native") return; + const process = cluster.resources.process; + if (process !== undefined) { + const running = yield* process.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (running) + yield* process.kill.pipe( + Effect.mapError((cause) => + ephemeralError("Unable to stop ephemeral Postgres", { cause }), + ), + ); + } + const scope = cluster.resources.processScope; + if (scope !== undefined) yield* Scope.close(scope, Exit.void); + cluster.resources.process = undefined; + cluster.resources.processScope = undefined; + cluster.running = false; + }); + +const stopContainer = (cluster: Cluster): Effect.Effect => + Effect.gen(function* () { + if (cluster.resources.kind !== "container") return; + const containerId = cluster.resources.containerId; + if (containerId !== undefined) + yield* cluster.resources.engine + .stopContainer(containerId) + .pipe( + Effect.mapError((cause) => + ephemeralError("Unable to stop ephemeral Postgres", { cause }), + ), + ); + cluster.running = false; + }); + +const startNative = ( + cluster: Cluster, + options: CreateEphemeralPostgresOptions, + healthTimeout: string, + password: string, +): Effect.Effect => + Effect.gen(function* () { + const executable = cluster.executable; + if (executable === undefined) + return yield* ephemeralError("Native Postgres executable is unavailable"); + const parentScope = yield* Scope.Scope; + const processScope = yield* Scope.fork(parentScope, "parallel"); + yield* Effect.uninterruptibleMask((restore) => + restore( + spawnNativeProcess( + { + executable, + args: postgresArgs(cluster.port, cluster.runtime, options.postgresSettings), + env: postgresEnv({ + port: cluster.port, + dataPath: cluster.dataPath, + password, + }), + cwd: cluster.root, + gracefulStopSignal: "SIGINT", + gracefulStopTimeout: "15 seconds", + }, + defaultNativeProcessLauncher(), + { stackId: cluster.identity, workloadId: DATABASE_WORKLOAD_ID }, + ).pipe(Scope.provide(processScope)), + ).pipe( + Effect.mapError((cause) => ephemeralError("Unable to start native Postgres", { cause })), + Effect.tap((process) => + Effect.sync(() => { + if (cluster.resources.kind === "native") { + cluster.resources.process = process; + cluster.resources.processScope = processScope; + } + }), + ), + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? Effect.void + : Scope.close(processScope, Exit.void).pipe(Effect.asVoid), + ), + ), + ); + yield* Effect.gen(function* () { + yield* waitForPostgres(cluster.port, healthTimeout); + if (!cluster.bootstrapped) { + yield* bootstrap(cluster.port, options, healthTimeout); + cluster.bootstrapped = true; + } + yield* waitForAdvertised(cluster.port, password, healthTimeout); + cluster.running = true; + }).pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) ? Effect.void : stopNative(cluster).pipe(Effect.ignore), + ), + ); + }); + +const startContainer = ( + cluster: Cluster, + options: CreateEphemeralPostgresOptions, + healthTimeout: string, + password: string, +): Effect.Effect => + Effect.gen(function* () { + if (cluster.resources.kind !== "container") return; + const resources = cluster.resources; + const image = cluster.image; + if (image === undefined) + return yield* ephemeralError("Ephemeral Postgres image is unavailable"); + const networkId = resources.networkId; + const volumeId = resources.volumeId; + if (networkId === undefined || volumeId === undefined) + return yield* ephemeralError("Ephemeral Postgres volume is unavailable"); + yield* Effect.gen(function* () { + if (resources.containerId !== undefined) { + yield* resources.engine + .startContainer(resources.containerId) + .pipe( + Effect.mapError((cause) => + ephemeralError("Unable to start ephemeral Postgres", { cause }), + ), + ); + } else { + const path = yield* Path.Path; + const envFile = yield* writeEnvFile( + path.join(cluster.root, "postgres.env"), + postgresEnv({ + port: 5432, + dataPath: `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}`, + password, + }), + ); + const created = yield* Effect.uninterruptibleMask((restore) => + restore( + resources.engine.createContainer({ + name: resourceName(cluster.identity, "database"), + image, + labels: { + stackId: cluster.identity, + ownerSessionId: cluster.identity.slice(0, 32), + workloadId: DATABASE_WORKLOAD_ID, + role: "workload", + }, + network: networkId, + mounts: [], + volumeMounts: [ + { + volume: volumeId, + target: `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}`, + readOnly: false, + }, + ], + publications: [{ address: "127.0.0.1", hostPort: cluster.port, containerPort: 5432 }], + role: "workload", + command: postgresArgs(5432, cluster.runtime, options.postgresSettings), + envFile, + networkAliases: [containerAliasFor("database:database")], + }), + ).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to create ephemeral Postgres", { cause }), + ), + Effect.tap((created) => + Effect.sync(() => { + resources.containerId = created.id; + }), + ), + ), + ); + yield* resources.engine + .startContainer(created.id) + .pipe( + Effect.mapError((cause) => + ephemeralError("Unable to start ephemeral Postgres", { cause }), + ), + ); + } + yield* waitForPostgres(cluster.port, healthTimeout); + if (!cluster.bootstrapped) { + yield* bootstrap(cluster.port, options, healthTimeout); + cluster.bootstrapped = true; + } + yield* waitForAdvertised(cluster.port, password, healthTimeout); + cluster.running = true; + }).pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) ? Effect.void : stopContainer(cluster).pipe(Effect.ignore), + ), + ); + }); + +const exportNative = ( + cluster: Cluster, + tarPath: string, +): Effect.Effect< + void, + EphemeralPostgresError, + ChildProcessSpawnerService | Scope.Scope | FileSystem.FileSystem +> => + Effect.gen(function* () { + yield* writeRuntimeMarker(cluster); + yield* runTar(["-C", cluster.root, "-cf", tarPath, PGDATA_DIR_NAME]); + }); + +const exportContainer = ( + cluster: Cluster, + tarPath: string, +): Effect.Effect => + Effect.gen(function* () { + yield* writeRuntimeMarker(cluster); + yield* runVolumeTar(cluster, tarPath, "create"); + }); + +const restoreNative = ( + cluster: Cluster, + restoreFrom: string, +): Effect.Effect< + void, + EphemeralPostgresError, + ChildProcessSpawnerService | Scope.Scope | FileSystem.FileSystem +> => + Effect.gen(function* () { + yield* runTar(["-C", cluster.root, "-xf", restoreFrom]); + yield* verifyRestoredMarker(cluster, restoreFrom); + }); + +const restoreContainer = ( + cluster: Cluster, + restoreFrom: string, +): Effect.Effect => + runVolumeTar(cluster, restoreFrom, "extract"); + +const peekSnapshotRuntime = ( + restoreFrom: string, + runtime: StackRuntime, + peekRoot: string, +): Effect.Effect< + void, + EphemeralPostgresError, + FileSystem.FileSystem | ChildProcessSpawnerService | Scope.Scope +> => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(peekRoot, { recursive: true }).pipe( + Effect.mapError((cause) => + ephemeralError("Unable to inspect snapshot", { + cause, + path: peekRoot, + reason: "snapshot", + }), + ), + ); + yield* runTar([ + "-xf", + restoreFrom, + "-C", + peekRoot, + `${PGDATA_DIR_NAME}/${RUNTIME_MARKER}`, + ]).pipe( + Effect.mapError(() => + ephemeralError("Ephemeral Postgres snapshot is missing a runtime marker", { + reason: "restore-mismatch", + path: restoreFrom, + }), + ), + ); + const marker = yield* fs + .readFileString(`${peekRoot}/${PGDATA_DIR_NAME}/${RUNTIME_MARKER}`) + .pipe( + Effect.mapError((cause) => + ephemeralError("Unable to read snapshot runtime marker", { cause, reason: "snapshot" }), + ), + Effect.flatMap(decodeMarker), + ); + if (!sameRuntime(marker, runtime)) + return yield* ephemeralError( + "Ephemeral Postgres snapshot was produced by a different runtime", + { + reason: "restore-mismatch", + path: restoreFrom, + }, + ); + yield* fs.remove(peekRoot, { recursive: true }).pipe(Effect.ignore); + }); + +const destroyCluster = (cluster: Cluster): Effect.Effect => + Effect.gen(function* () { + if (cluster.resources.kind === "native") yield* stopNative(cluster).pipe(Effect.ignore); + else { + yield* stopContainer(cluster).pipe(Effect.ignore); + if (cluster.resources.containerId !== undefined) + yield* cluster.resources.engine + .removeContainer(cluster.resources.containerId) + .pipe(Effect.ignore); + if (cluster.resources.volumeId !== undefined) + yield* cluster.resources.engine + .removeVolume(cluster.resources.volumeId) + .pipe(Effect.ignore); + if (cluster.resources.networkId !== undefined) + yield* cluster.resources.engine + .removeNetwork(cluster.resources.networkId) + .pipe(Effect.ignore); + } + const fs = yield* FileSystem.FileSystem; + yield* fs.remove(cluster.root, { recursive: true }).pipe(Effect.ignore); + }); + +const clusterHandle = ( + cluster: Cluster, + options: CreateEphemeralPostgresOptions, + healthTimeout: string, + password: string, +): EffectEphemeralPostgres => { + const requireStopped = (): Effect.Effect => + cluster.running + ? Effect.fail( + ephemeralError("Ephemeral Postgres must be stopped before exporting PGDATA", { + reason: "not-stopped", + }), + ) + : Effect.void; + return { + host: cluster.host, + port: cluster.port, + version: cluster.version, + runtime: cluster.runtime, + artifactIdentity: cluster.artifactIdentity, + url: Redacted.make(databaseUrl(cluster.port, password)), + ...(cluster.resources.kind === "container" && cluster.resources.networkId !== undefined + ? { networkId: cluster.resources.networkId } + : {}), + start: Effect.suspend(() => + cluster.lifecycle.withPermit( + Effect.gen(function* () { + if (cluster.running) { + if (cluster.runtime.kind === "native" && cluster.resources.kind === "native") { + const process = cluster.resources.process; + const stillRunning = + process === undefined + ? false + : yield* process.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (stillRunning) return; + yield* stopNative(cluster).pipe(Effect.ignore); + } else { + const probe = yield* waitForPostgres(cluster.port, "1s").pipe(Effect.exit); + if (Exit.isSuccess(probe)) return; + cluster.running = false; + } + } + if (cluster.runtime.kind === "native") + yield* startNative(cluster, options, healthTimeout, password); + else yield* startContainer(cluster, options, healthTimeout, password); + }), + ), + ), + stop: Effect.suspend(() => + cluster.lifecycle.withPermit( + cluster.runtime.kind === "native" ? stopNative(cluster) : stopContainer(cluster), + ), + ), + exportPgData: (tarPath) => + cluster.lifecycle.withPermit( + Effect.gen(function* () { + yield* requireStopped(); + if (cluster.runtime.kind === "native") yield* exportNative(cluster, tarPath); + else yield* exportContainer(cluster, tarPath); + }), + ), + }; +}; + +export const createEphemeralPostgresCluster = ( + options: CreateEphemeralPostgresOptions, +): Effect.Effect< + EffectEphemeralPostgres, + EphemeralPostgresCreateError, + Scope.Scope | FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService +> => + Effect.gen(function* () { + const resolver = yield* Effect.serviceOption(ContainerEngineResolver).pipe( + Effect.map(Option.getOrUndefined), + ); + const runtime = yield* resolvedRuntime(options.runtime, resolver); + const release = yield* resolveEphemeralPostgresRelease(options.version); + const env = yield* Effect.serviceOption(StackRuntimeEnvironment).pipe( + Effect.flatMap((configured) => + Option.isSome(configured) ? Effect.succeed(configured.value) : defaultRuntimeEnvironment, + ), + ); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const identity = yield* createIdentity(crypto); + const root = path.join(path.dirname(env.stateRoot), "ephemeral-postgres", identity); + const dataPath = path.join(root, PGDATA_DIR_NAME); + yield* fs.makeDirectory(dataPath, { recursive: true, mode: 0o700 }).pipe( + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Unable to create ephemeral Postgres data directory", + path: dataPath, + cause, + }), + ), + ); + yield* Effect.addFinalizer(() => fs.remove(root, { recursive: true }).pipe(Effect.ignore)); + if (options.restoreFrom !== undefined) + yield* peekSnapshotRuntime(options.restoreFrom, runtime, path.join(root, "peek")); + const port = yield* allocateLoopbackPort(options.port); + const healthTimeout = options.healthTimeout ?? DEFAULT_DATABASE_HEALTH_TIMEOUT; + const password = Redacted.value(options.databasePassword); + const workload = plannedWorkload(release.version, release.image, runtime); + const preparer = yield* makeProductionRuntimeArtifactPreparer({ + stateRoot: env.stateRoot, + ...(env.artifactCacheRoot === undefined ? {} : { artifactCacheRoot: env.artifactCacheRoot }), + runtime, + }); + const prepared = yield* preparer.prepare(runtime, workload); + let resources: RuntimeResources; + if (runtime.kind === "native") { + resources = { kind: "native" }; + } else { + const engine = yield* resolveContainerEngine(runtime.engine, resolver).pipe( + Effect.mapError( + (cause) => + new ContainerEngineError({ + message: `Unable to configure ${runtime.engine} for ephemeral Postgres`, + engine: runtime.engine, + cause, + }), + ), + ); + resources = { kind: "container", engine }; + } + const cluster: Cluster = { + identity, + root, + dataPath, + host: "127.0.0.1", + port, + version: release.version, + runtime, + artifactIdentity: + runtime.kind === "native" + ? `native:${release.version}` + : `container:${runtime.engine}:${release.image}`, + ...(prepared.executablePath === undefined || prepared.artifactRoot === undefined + ? {} + : { + executable: prepared.artifactRoot.endsWith("/") + ? `${prepared.artifactRoot}${prepared.executablePath}` + : `${prepared.artifactRoot}/${prepared.executablePath}`, + }), + ...(prepared.image === undefined ? {} : { image: prepared.image }), + lifecycle: Semaphore.makeUnsafe(1), + running: false, + bootstrapped: options.restoreFrom !== undefined, + resources, + }; + yield* Effect.addFinalizer(() => destroyCluster(cluster)); + if (cluster.resources.kind === "container") { + const resources = cluster.resources; + const engine = resources.engine; + const engineKind = cluster.runtime.kind === "container" ? cluster.runtime.engine : "docker"; + yield* Effect.uninterruptibleMask((restore) => + restore( + engine.createNetwork({ + name: resourceName(identity, "network"), + labels: { stackId: identity, ownerSessionId: identity.slice(0, 32), role: "network" }, + }), + ).pipe( + Effect.mapError( + (cause) => + new ContainerEngineError({ + message: "Unable to create ephemeral Postgres network", + engine: engineKind, + cause, + }), + ), + Effect.tap((created) => + Effect.sync(() => { + resources.networkId = created.id; + }), + ), + ), + ); + yield* Effect.uninterruptibleMask((restore) => + restore( + engine.createVolume({ + name: resourceName(identity, "database-volume"), + labels: { stackId: identity, workloadId: DATABASE_WORKLOAD_ID, role: "volume" }, + }), + ).pipe( + Effect.mapError( + (cause) => + new ContainerEngineError({ + message: "Unable to create ephemeral Postgres volume", + engine: engineKind, + cause, + }), + ), + Effect.tap((created) => + Effect.sync(() => { + resources.volumeId = created.id; + }), + ), + ), + ); + } + if (options.restoreFrom !== undefined) { + if (runtime.kind === "native") yield* restoreNative(cluster, options.restoreFrom); + else yield* restoreContainer(cluster, options.restoreFrom); + } + if (runtime.kind === "native") yield* startNative(cluster, options, healthTimeout, password); + else yield* startContainer(cluster, options, healthTimeout, password); + return clusterHandle(cluster, options, healthTimeout, password); + }); diff --git a/packages/stack/src/runtime/NativeRuntime.ts b/packages/stack/src/runtime/NativeRuntime.ts index c08a8c0f0e..c26237d0ec 100644 --- a/packages/stack/src/runtime/NativeRuntime.ts +++ b/packages/stack/src/runtime/NativeRuntime.ts @@ -48,6 +48,8 @@ export interface NativeRuntimeOptions { process?: NativeProcess, ) => Effect.Effect; readonly logStore?: LogStore; + /** Wipes native PGDATA after the database workload has been stopped and removed. */ + readonly wipeDatabaseData?: Effect.Effect; } /** One-shot startup processes followed by the long-lived workload process. */ @@ -593,11 +595,19 @@ export const makeNativeRuntime = ( }), ); + const wipePersistentData = ( + key: RuntimeWorkloadKey, + ): Effect.Effect => + key.workloadId === "database:database" && options.wipeDatabaseData !== undefined + ? options.wipeDatabaseData + : Effect.void; + return { observe, start, stop, remove, cleanup: cleanupRuntime, + wipePersistentData, } satisfies RuntimeDriver; }); diff --git a/packages/stack/src/runtime/PostgresDatabaseSession.ts b/packages/stack/src/runtime/PostgresDatabaseSession.ts index 0a313b1515..c5cf9a4a68 100644 --- a/packages/stack/src/runtime/PostgresDatabaseSession.ts +++ b/packages/stack/src/runtime/PostgresDatabaseSession.ts @@ -3,6 +3,10 @@ import { Context, Duration, Effect, Layer, Predicate, Redacted, Schema, Scope } import { isSqlError, type SqlError } from "effect/unstable/sql/SqlError"; import { DatabaseBootstrapError, + INTERNAL_DATABASE, + INTERNAL_SCHEMAS, + JWT_SECRET_SETTING, + type DatabaseBootstrapOptions, type DatabaseSession, type DatabaseSqlValue, type DatabaseTransaction, @@ -105,7 +109,7 @@ export const makeDatabaseSessionFromSqlClient = ( .join(", "); const parameters = settings.flatMap((setting) => [ setting.name, - setting.name === "app.settings.jwt_secret" ? Redacted.value(setting.value) : setting.value, + setting.name === JWT_SECRET_SETTING ? Redacted.value(setting.value) : setting.value, ]); return generated( `SELECT string_agg(format('ALTER DATABASE postgres SET %I TO %L', name, value), E';\\n') AS statement FROM (VALUES ${values}) AS settings(name, value)`, @@ -154,9 +158,6 @@ const makePostgresDatabaseSession = ( }), ); -const INTERNAL_DATABASE = "_supabase"; -const INTERNAL_SCHEMAS = ["_analytics", "_supavisor"] as const; - /** * Ensures the private database and service-owned schemas exist before any * dependent workload is started. Database creation happens outside a transaction because @@ -214,3 +215,30 @@ export const bootstrapDatabaseAt = ( }), ); }); + +/** Reconciles roles, JWT settings, and `_supabase` against an already-ready Postgres. */ +export const bootstrapManagedPostgres = ( + options: DatabaseBootstrapOptions & { + readonly host: string; + readonly port: number; + }, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const session = yield* makePostgresDatabaseSession({ + host: options.host, + port: options.port, + password: options.databasePassword, + }); + yield* ensureInternalDatabase( + session, + makePostgresDatabaseSession({ + host: options.host, + port: options.port, + database: INTERNAL_DATABASE, + password: options.databasePassword, + }), + ); + yield* runDatabaseBootstrap(session, options); + }), + ); diff --git a/packages/stack/src/runtime/ProductionRuntime.ts b/packages/stack/src/runtime/ProductionRuntime.ts index 751e9d4bb5..5e61e59013 100644 --- a/packages/stack/src/runtime/ProductionRuntime.ts +++ b/packages/stack/src/runtime/ProductionRuntime.ts @@ -967,6 +967,26 @@ export const makeProductionRuntime = ( waitForReadiness, bootstrapDatabase: bootstrapWorkloadDatabase, logStore: logs, + wipeDatabaseData: Effect.gen(function* () { + const dataPath = pathService.join(paths.data, "database"); + const key = { stackId: options.stackId, workloadId: "database:database" }; + const exists = yield* fileSystem.exists(dataPath).pipe(Effect.orElseSucceed(() => false)); + if (exists) + yield* fileSystem + .remove(dataPath, { recursive: true }) + .pipe( + Effect.mapError((error) => + driverError(key, "Unable to wipe native database data", error), + ), + ); + yield* fileSystem + .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) + .pipe( + Effect.mapError((error) => + driverError(key, "Unable to recreate native database data directory", error), + ), + ); + }), }).pipe( Effect.mapError((error) => preparationError("Unable to initialize native runtime", error)), ); diff --git a/packages/stack/src/runtime/RuntimeDriver.ts b/packages/stack/src/runtime/RuntimeDriver.ts index 55fef48c5c..30f4285303 100644 --- a/packages/stack/src/runtime/RuntimeDriver.ts +++ b/packages/stack/src/runtime/RuntimeDriver.ts @@ -46,6 +46,11 @@ export interface RuntimeDriver { * cleanup removes them after containers and networks have been removed. */ readonly cleanup: (request: RuntimeCleanupRequest) => Effect.Effect; + /** + * Wipes persistent data for one stopped and removed workload. Native database data directories + * and container volumes owned by that workload are removed; other stack volumes stay. + */ + readonly wipePersistentData: (key: RuntimeWorkloadKey) => Effect.Effect; } export class RuntimeDriverError extends Data.TaggedError("RuntimeDriverError")<{ diff --git a/packages/stack/src/runtime/RuntimeEnvFile.ts b/packages/stack/src/runtime/RuntimeEnvFile.ts index b91a12059d..a89f25fffd 100644 --- a/packages/stack/src/runtime/RuntimeEnvFile.ts +++ b/packages/stack/src/runtime/RuntimeEnvFile.ts @@ -41,7 +41,7 @@ const mapFile = ( ), ); -const contentFor = ( +export const encodeRuntimeEnvFile = ( values: Readonly>, ): Effect.Effect => { const entries = Object.entries(values).sort(([left], [right]) => left.localeCompare(right)); @@ -81,7 +81,7 @@ export const makeRuntimeEnvFileOwner = ( if (!validWorkloadId(input.workloadId)) return Effect.fail(error("Invalid runtime environment workload identity")); return Effect.gen(function* () { - const text = yield* contentFor(input.values); + const text = yield* encodeRuntimeEnvFile(input.values); const target = path.join(envRoot, `${encodeWorkloadId(input.workloadId)}.env`); const token = yield* crypto.randomUUIDv4.pipe( Effect.mapError(() => error("Unable to allocate runtime environment file name")), diff --git a/packages/stack/src/runtime/RuntimeEnvFile.unit.test.ts b/packages/stack/src/runtime/RuntimeEnvFile.unit.test.ts new file mode 100644 index 0000000000..edc85189f4 --- /dev/null +++ b/packages/stack/src/runtime/RuntimeEnvFile.unit.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, Option } from "effect"; +import { StackPreparationError } from "../public/Errors.ts"; +import { encodeRuntimeEnvFile } from "./RuntimeEnvFile.ts"; + +describe("encodeRuntimeEnvFile", () => { + it.effect("encodes sorted NAME=value lines", () => + Effect.gen(function* () { + const text = yield* encodeRuntimeEnvFile({ ZETA: "2", ALPHA: "1" }); + expect(text).toBe("ALPHA=1\nZETA=2\n"); + }), + ); + + it.effect("rejects CR/LF in a value", () => + Effect.gen(function* () { + const exit = yield* encodeRuntimeEnvFile({ POSTGRES_PASSWORD: "x\ny" }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + expect(error).toBeInstanceOf(StackPreparationError); + if (!(error instanceof StackPreparationError)) return; + expect(error.message).toBe("Invalid runtime environment variable value"); + }), + ); + + it.effect("rejects CR/LF in a name", () => + Effect.gen(function* () { + const exit = yield* encodeRuntimeEnvFile({ "FOO\nBAR": "1" }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + expect(error).toBeInstanceOf(StackPreparationError); + if (!(error instanceof StackPreparationError)) return; + expect(error.message).toBe("Invalid runtime environment variable name"); + }), + ); +}); diff --git a/packages/stack/src/runtime/SchemaInit.ts b/packages/stack/src/runtime/SchemaInit.ts new file mode 100644 index 0000000000..148ee63307 --- /dev/null +++ b/packages/stack/src/runtime/SchemaInit.ts @@ -0,0 +1,640 @@ +import { + Crypto, + Duration, + Effect, + FileSystem, + Option, + Path, + Redacted, + Schema, + Scope, + Stream, +} from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { compileStack } from "../model/Compiler.ts"; +import { excludeStackCapabilities, type ExcludableCapabilityName } from "../model/Exclusions.ts"; +import { CAPABILITY_NAMES } from "../public/Capability.ts"; +import type { + SchemaInitCapabilityName, + SchemaInitOptions, + SchemaInitTarget, +} from "../public/SchemaInit.ts"; +import type { SchemaInitError } from "../public/Errors.ts"; +import { StackIdSchema, type StackId } from "../public/StackId.ts"; +import { + ContainerEngineError, + InvalidStackConfigError, + InvalidStackIdentityError, + RequiresActivatedProcessError, + StackPreparationError, + StackRuntimeError, +} from "../public/Errors.ts"; +import type { StackRuntime } from "../public/Runtime.ts"; +import { + AUTH_JWT_SECRET_SLOT, + DATABASE_INTERNAL_PASSWORD_SLOT, + resolveSecrets, +} from "../state/SecretStore.ts"; +import { STACK_STATE_FORMAT, type PersistedStackState } from "../state/StackState.ts"; +import { defaultRuntimeEnvironment } from "../supervisor/Launcher.ts"; +import { makeProductionRuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; +import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; +import type { ContainerEngine, ContainerHostRoute } from "./ContainerEngine.ts"; +import { resolveContainerEngine } from "./ContainerEngineResolver.ts"; +import { runContainerStartupProcess, schemaInitContainerName } from "./ContainerRuntime.ts"; +import { + defaultNativeProcessLauncher, + spawnNativeProcess, + type NativeProcessSpec, +} from "./NativeProcess.ts"; +import { makeRuntimeInputOwner } from "./RuntimeInputOwner.ts"; +import { encodeRuntimeEnvFile } from "./RuntimeEnvFile.ts"; +import type { RuntimeWorkloadKey } from "./RuntimeDriver.ts"; +import { + runtimeSpecFor, + validateWorkloadRuntimeInputs, + type WorkloadRuntimeInputs, + type WorkloadRuntimeSpec, +} from "./WorkloadRuntimeSpec.ts"; + +const STARTUP_TIMEOUT = "5 minutes" satisfies Duration.Input; +const AUTH_TEMPLATE_BASE_URL = "http://127.0.0.1"; + +const PRIMARY_WORKLOAD: Record = { + auth: "auth:auth", + storage: "storage:storage", + realtime: "realtime:realtime", + analytics: "analytics:analytics", + pooler: "pooler:pooler", +}; + +const SCHEMA_INIT_BINDINGS = ["primary", "admin", "ui", "smtp", "pop3", "inspector"] as const; + +const schemaInitCompileConfig = ( + config: SchemaInitTarget["config"], + names: ReadonlyArray, +) => { + const requested = new Set(names); + return excludeStackCapabilities( + config, + CAPABILITY_NAMES.filter( + (name): name is ExcludableCapabilityName => name !== "database" && !requested.has(name), + ), + ); +}; + +/** Catalog version and image for a schema-init one-shot; undefined when the pin is unknown. */ +export const schemaInitArtifactIdentity = ( + name: SchemaInitCapabilityName, + version?: string, +): string | undefined => { + const release = catalogReleaseFor(PRIMARY_WORKLOAD[name], version); + return release === undefined ? undefined : `${release.version}:${release.containerImage}`; +}; + +const schemaInitPrivatePorts = ( + databasePort: number, + workloadId: string, + bindings: WorkloadRuntimeSpec["bindings"], +): PersistedStackState["privatePorts"] => [ + { workloadId: "database:database", binding: "primary", port: databasePort }, + ...SCHEMA_INIT_BINDINGS.flatMap((binding) => { + const bound = bindings[binding]; + return bound === undefined ? [] : [{ workloadId, binding, port: bound.containerPort }]; + }), +]; + +/** Linux Engine extra hosts so `host.docker.internal` resolves (DNS even when URLs stay on the alias). */ +export const schemaInitHostGatewayExtraHosts = ( + platform: string, + host: string, +): ReadonlyArray => + platform === "linux" && host === "host.docker.internal" + ? ["host.docker.internal:host-gateway"] + : []; + +const DATABASE_HOST_KEYS = new Set([ + "DB_HOST", + "GOTRUE_DB_HOST", + "POSTGRES_HOST", + "PGHOST", + "PG_META_DB_HOST", +]); +const DATABASE_PORT_KEYS = new Set([ + "DB_PORT", + "GOTRUE_DB_PORT", + "POSTGRES_PORT", + "PGPORT", + "PG_META_DB_PORT", +]); +const DATABASE_PASSWORD_KEYS = new Set([ + "DB_PASSWORD", + "GOTRUE_DB_PASSWORD", + "POSTGRES_PASSWORD", + "PGPASSWORD", + "PG_META_DB_PASSWORD", +]); +const CONNECTION_PROTOCOLS = new Set(["postgres:", "postgresql:", "ecto:"]); + +export interface ParsedDatabaseUrl { + readonly host: string; + readonly port: number; + readonly password: string; + readonly database: string; +} + +export const parseSchemaInitDatabaseUrl = (url: string): ParsedDatabaseUrl | undefined => { + try { + const parsed = new URL(url); + if (!CONNECTION_PROTOCOLS.has(parsed.protocol)) return undefined; + const port = Number.parseInt(parsed.port === "" ? "5432" : parsed.port, 10); + if (!Number.isInteger(port) || parsed.hostname.length === 0) return undefined; + const database = decodeURIComponent(parsed.pathname.replace(/^\//, "")); + return { + host: parsed.hostname, + port, + password: decodeURIComponent(parsed.password), + database, + }; + } catch { + return undefined; + } +}; + +const rewriteConnectionUrl = ( + value: string, + target: { readonly host: string; readonly port: number; readonly password: string }, +): string | undefined => { + if (!value.includes("://")) return undefined; + try { + const parsed = new URL(value); + if (!CONNECTION_PROTOCOLS.has(parsed.protocol)) return undefined; + parsed.hostname = target.host; + parsed.port = String(target.port); + parsed.password = target.password; + return parsed.toString(); + } catch { + return undefined; + } +}; + +/** Rewrites only host/port/password on database connection settings; keeps user and database name. */ +export const rewriteDatabaseEnvironment = ( + env: Readonly>, + target: { readonly host: string; readonly port: number; readonly password: string }, +): Record => { + const rewritten: Record = {}; + for (const [key, value] of Object.entries(env)) { + if (DATABASE_HOST_KEYS.has(key)) { + rewritten[key] = target.host; + continue; + } + if (DATABASE_PORT_KEYS.has(key)) { + rewritten[key] = String(target.port); + continue; + } + if (DATABASE_PASSWORD_KEYS.has(key)) { + rewritten[key] = target.password; + continue; + } + rewritten[key] = rewriteConnectionUrl(value, target) ?? value; + } + return rewritten; +}; + +const loopbackHost = (host: string): boolean => host === "127.0.0.1" || host === "localhost"; + +const schemaInitIdentity = ( + crypto: Crypto.Crypto, +): Effect.Effect => + Effect.gen(function* () { + const first = yield* crypto.randomUUIDv4; + const second = yield* crypto.randomUUIDv4; + return yield* Schema.decodeEffect(StackIdSchema)(`${first}${second}`.replaceAll("-", "")); + }).pipe( + Effect.mapError( + (cause) => + new InvalidStackIdentityError({ + message: "Unable to allocate schema-init identity", + cause, + }), + ), + ); + +const runtimeError = ( + key: Pick, + message: string, + cause?: unknown, +): StackRuntimeError => + new StackRuntimeError({ + message, + stackId: key.stackId, + workloadId: key.workloadId, + ...(cause === undefined ? {} : { cause }), + }); + +const mapContainerEngineError = ( + engine: StackRuntime & { readonly kind: "container" }, + message: string, + cause: unknown, +): ContainerEngineError => + new ContainerEngineError({ + engine: engine.engine, + message, + cause, + }); + +const resolveEngine = ( + target: SchemaInitTarget, + options: SchemaInitOptions, +): Effect.Effect< + Option.Option, + ContainerEngineError, + ChildProcessSpawner.ChildProcessSpawner +> => { + if (target.runtime.kind !== "container") return Effect.succeed(Option.none()); + if (options.containerEngine !== undefined) + return Effect.succeed(Option.some(options.containerEngine)); + const runtime = target.runtime; + return resolveContainerEngine(runtime.engine).pipe( + Effect.map(Option.some), + Effect.mapError((cause) => + mapContainerEngineError( + runtime, + `Unable to configure ${runtime.engine} for schema init`, + cause, + ), + ), + ); +}; + +const resolveLiveNetwork = ( + engine: ContainerEngine, + stackId: StackId, + runtime: StackRuntime & { readonly kind: "container" }, +): Effect.Effect => + engine.listResources(stackId).pipe( + Effect.mapError((cause) => + mapContainerEngineError(runtime, "Unable to list stack resources for schema init", cause), + ), + Effect.flatMap((resources) => { + const network = resources.find((entry) => entry.kind === "network"); + return network === undefined + ? Effect.fail( + runtimeError( + { stackId, workloadId: "" }, + "Stack network is unavailable for schema init", + ), + ) + : Effect.succeed(network.id); + }), + ); + +const acquireEphemeralNetwork = ( + engine: ContainerEngine, + schemaInitId: StackId, + runtime: StackRuntime & { readonly kind: "container" }, +): Effect.Effect => + Effect.gen(function* () { + const created = yield* engine + .createNetwork({ + name: `supabase-${schemaInitId.slice(0, 16)}-schema-init-net`, + labels: { + stackId: schemaInitId, + ownerSessionId: schemaInitId.slice(0, 32), + role: "network", + }, + }) + .pipe( + Effect.mapError((cause) => + mapContainerEngineError(runtime, "Unable to create schema-init network", cause), + ), + ); + yield* Effect.addFinalizer(() => engine.removeNetwork(created.id).pipe(Effect.ignore)); + return created.id; + }); + +const runNativeStartup = ( + spec: NativeProcessSpec, + key: RuntimeWorkloadKey, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const process = yield* spawnNativeProcess(spec, defaultNativeProcessLauncher(), key).pipe( + Effect.mapError((error) => runtimeError(key, error.message, error)), + ); + const drain = Effect.all([Stream.runDrain(process.stdout), Stream.runDrain(process.stderr)], { + concurrency: "unbounded", + discard: true, + }).pipe(Effect.mapError((error) => runtimeError(key, error.message, error))); + const exit = process.exitCode.pipe( + Effect.mapError((error) => runtimeError(key, error.message, error)), + ); + const [exitCode] = yield* Effect.all([exit, drain], { concurrency: "unbounded" }).pipe( + Effect.timeoutOrElse({ + duration: spec.timeout ?? STARTUP_TIMEOUT, + orElse: () => + Effect.fail(runtimeError(key, `Native schema init timed out for ${key.workloadId}`)), + }), + ); + if (exitCode !== 0) + return yield* runtimeError( + key, + `Native schema init exited with code ${String(exitCode)} for ${key.workloadId}`, + ); + }), + ); + +const capabilityInputs = ( + material: WorkloadRuntimeInputs, + hostRoute: ContainerHostRoute | undefined, +): WorkloadRuntimeInputs => { + const templateBaseUrl = material.auth?.templateBaseUrl ?? AUTH_TEMPLATE_BASE_URL; + const auth = { + ...material.auth, + templateBaseUrl, + }; + return { + ...material, + auth, + ...(hostRoute === undefined ? {} : { hostRoute }), + }; +}; + +export const schemaInitWorkloads = ( + names: ReadonlyArray, + target: SchemaInitTarget, + options: SchemaInitOptions = {}, +): Effect.Effect< + void, + SchemaInitError, + FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawner.ChildProcessSpawner +> => + Effect.scoped( + Effect.gen(function* () { + const parsed = parseSchemaInitDatabaseUrl(target.databaseUrl); + if (parsed === undefined) + return yield* new InvalidStackConfigError({ + message: "Schema init requires a valid PostgreSQL URL", + }); + const password = Redacted.value(target.secrets.databasePassword); + const connection = { host: parsed.host, port: parsed.port, password }; + const compiled = yield* compileStack({ + projectRoot: target.projectRoot, + runtime: target.runtime, + config: schemaInitCompileConfig(target.config, names), + }); + const declarations = compiled.secrets.map((entry) => { + if (entry.slot === DATABASE_INTERNAL_PASSWORD_SLOT) + return { ...entry, value: target.secrets.databasePassword }; + if (entry.slot === AUTH_JWT_SECRET_SLOT && target.secrets.jwtSecret !== undefined) + return { ...entry, value: target.secrets.jwtSecret }; + return entry; + }); + const resolved = yield* resolveSecrets({ declarations }, undefined, "unconfigured"); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const schemaInitId = yield* schemaInitIdentity(crypto); + const tempRoot = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-schema-init-" }).pipe( + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Unable to create schema-init workspace", + cause, + }), + ), + ); + const shared = yield* defaultRuntimeEnvironment; + const engine = Option.getOrUndefined(yield* resolveEngine(target, options)); + const preparer = + options.artifactPreparer ?? + (yield* makeProductionRuntimeArtifactPreparer({ + stateRoot: shared.stateRoot, + ...(shared.artifactCacheRoot === undefined + ? {} + : { artifactCacheRoot: shared.artifactCacheRoot }), + runtime: target.runtime, + ...(engine === undefined ? {} : { containerEngine: engine }), + })); + const inputOwner = yield* makeRuntimeInputOwner({ + stateRoot: tempRoot, + stackId: schemaInitId, + }); + yield* Effect.addFinalizer(() => inputOwner.cleanupAll.pipe(Effect.ignore)); + const joinPostgresNetwork = target.kind === "ephemeral" && target.networkId !== undefined; + const envKind = + target.runtime.kind === "container" && (target.kind === "live" || joinPostgresNetwork) + ? "container" + : "native"; + const privatePorts: PersistedStackState["privatePorts"] = [ + { workloadId: "database:database", binding: "primary", port: connection.port }, + ]; + const state: PersistedStackState = { + format: STACK_STATE_FORMAT, + identity: { + projectRoot: target.projectRoot, + branchContext: "schema-init", + stackName: "schema-init", + }, + runtime: target.runtime, + desiredLifecycle: "stopped", + definition: compiled.definition, + ports: [], + privatePorts, + secrets: resolved.persisted, + }; + let hostRoute: ContainerHostRoute | undefined; + let networkId: string | undefined; + if (target.runtime.kind === "container") { + const runtime = target.runtime; + if (engine === undefined) + return yield* new ContainerEngineError({ + engine: runtime.engine, + message: "Container engine is unavailable for schema init", + }); + hostRoute = yield* engine.preflight.pipe( + Effect.mapError((cause) => + mapContainerEngineError(runtime, "Container host route preflight failed", cause), + ), + ); + networkId = + target.kind === "live" + ? yield* resolveLiveNetwork(engine, target.stackId, runtime) + : target.networkId !== undefined + ? target.networkId + : yield* acquireEphemeralNetwork(engine, schemaInitId, runtime); + } + const rewriteTarget = + target.runtime.kind === "container" && target.kind === "ephemeral" && !joinPostgresNetwork + ? { + host: + hostRoute !== undefined && loopbackHost(connection.host) + ? hostRoute.host + : connection.host, + port: connection.port, + password, + } + : undefined; + const extraHosts = + target.runtime.kind === "container" && target.kind === "ephemeral" + ? schemaInitHostGatewayExtraHosts( + options.platform ?? process.platform, + hostRoute?.host ?? "host.docker.internal", + ) + : []; + yield* Effect.forEach( + names, + (name) => + Effect.gen(function* () { + if (compiled.definition.capabilities[name].enabled !== true) { + yield* Effect.logWarning(`Skipping schema init for disabled capability ${name}`); + return; + } + const workloadId = PRIMARY_WORKLOAD[name]; + const workload = compiled.executionPlan.workloads.find( + (entry) => entry.id === workloadId, + ); + if (workload === undefined) + return yield* runtimeError( + { stackId: schemaInitId, workloadId }, + `Missing planned workload for ${name} schema init`, + ); + const spec = runtimeSpecFor(workload); + if (spec === undefined) + return yield* new StackPreparationError({ + message: `Unknown runtime specification for ${workload.id}`, + workload: workload.id, + }); + const key: RuntimeWorkloadKey = { stackId: schemaInitId, workloadId }; + const dummyPort = spec.containerPort; + const envState: PersistedStackState = { + ...state, + privatePorts: schemaInitPrivatePorts(connection.port, workloadId, spec.bindings), + }; + const material = yield* inputOwner.resolve(envState, workload.id); + const inputs = capabilityInputs(material, hostRoute); + yield* validateWorkloadRuntimeInputs(envState, workload, inputs); + const environment = yield* Effect.try({ + try: () => spec.env(envState, workload, dummyPort, envKind, inputs), + catch: (cause) => + runtimeError(key, cause instanceof Error ? cause.message : String(cause), cause), + }); + const env = + rewriteTarget === undefined + ? environment + : rewriteDatabaseEnvironment(environment, rewriteTarget); + if (target.runtime.kind === "native") { + const preview = spec.nativeStartupProcesses( + "", + envState, + workload, + dummyPort, + inputs, + ); + if (preview.length === 0) + return yield* new RequiresActivatedProcessError({ + capability: name, + message: `${name} schema init requires the activated ${name} process`, + }); + const prepared = yield* preparer.prepare(target.runtime, workload); + if (prepared.artifactRoot === undefined) + return yield* runtimeError( + key, + `Native artifact root is unavailable for ${workload.id}`, + ); + const startups = spec.nativeStartupProcesses( + prepared.artifactRoot, + envState, + workload, + dummyPort, + inputs, + ); + yield* Effect.forEach( + startups, + (startup) => + runNativeStartup( + { + ...startup, + timeout: STARTUP_TIMEOUT, + env: { ...env, ...startup.env }, + }, + key, + ), + { discard: true }, + ); + return; + } + const startups = spec.containerStartupProcesses(envState, workload, inputs); + if (startups.length === 0) + return yield* new RequiresActivatedProcessError({ + capability: name, + message: `${name} schema init requires the activated ${name} process`, + }); + if (engine === undefined || networkId === undefined) + return yield* new ContainerEngineError({ + engine: target.runtime.kind === "container" ? target.runtime.engine : "docker", + message: "Container engine is unavailable for schema init", + }); + const prepared = yield* preparer.prepare(target.runtime, workload); + if (prepared.image === undefined) + return yield* runtimeError(key, `Container image is unavailable for ${workload.id}`); + const encoded = yield* encodeRuntimeEnvFile(env); + const envFile = path.join(tempRoot, `${encodeURIComponent(workload.id)}.env`); + yield* fs.writeFileString(envFile, encoded).pipe( + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Unable to write schema-init environment file", + path: envFile, + cause, + }), + ), + ); + yield* fs.chmod(envFile, 0o600).pipe( + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Unable to secure schema-init environment file", + path: envFile, + cause, + }), + ), + ); + const image = prepared.image; + const mounts = spec.containerMounts?.(envState, workload, inputs) ?? []; + yield* Effect.forEach( + startups, + (startup) => + runContainerStartupProcess({ + engine, + key, + timeout: STARTUP_TIMEOUT, + specification: { + name: schemaInitContainerName(key), + image, + labels: { + stackId: schemaInitId, + ownerSessionId: schemaInitId.slice(0, 32), + workloadId, + startup: true, + role: "workload", + }, + network: networkId, + mounts, + volumeMounts: [], + publications: [], + role: "workload", + entrypoint: startup.entrypoint, + command: startup.command, + envFile, + ...(extraHosts.length === 0 ? {} : { extraHosts }), + }, + }).pipe(Effect.mapError((error) => runtimeError(key, error.message, error))), + { discard: true }, + ); + }), + { discard: true }, + ); + }), + ); diff --git a/packages/stack/src/runtime/WorkloadRuntimeSpec.ts b/packages/stack/src/runtime/WorkloadRuntimeSpec.ts index 9bb0253ada..b4703f00b9 100644 --- a/packages/stack/src/runtime/WorkloadRuntimeSpec.ts +++ b/packages/stack/src/runtime/WorkloadRuntimeSpec.ts @@ -1109,8 +1109,7 @@ const specs: Readonly> = { ? `http://${containerAliasFor("analytics:analytics")}:4000` : `http://127.0.0.1:${workloadPort(state, "analytics:analytics", "primary", runtime, 4000)}`, LOGFLARE_PRIVATE_ACCESS_TOKEN: valueAt(state, "analytics", "api_key"), - NEXT_PUBLIC_ENABLE_LOGS: - valueAt(state, "analytics", "backend").length > 0 ? "true" : "false", + NEXT_PUBLIC_ENABLE_LOGS: capabilityEnabled(state, "analytics") ? "true" : "false", NEXT_ANALYTICS_BACKEND_PROVIDER: valueAt(state, "analytics", "backend"), SUPABASE_URL: apiGatewayUrl(state, runtime === "container" ? inputs : undefined), SUPABASE_PUBLIC_URL: apiListenerUrl(state), diff --git a/packages/stack/src/runtime/production-runtime.integration.test.ts b/packages/stack/src/runtime/production-runtime.integration.test.ts index f927a96e4f..5a0c5518e6 100644 --- a/packages/stack/src/runtime/production-runtime.integration.test.ts +++ b/packages/stack/src/runtime/production-runtime.integration.test.ts @@ -2902,6 +2902,7 @@ describe("production runtime", () => { stop: () => Effect.void, remove: () => Effect.void, cleanup: () => Effect.fail(runtimeFailure), + wipePersistentData: () => Effect.void, }; const envOwner: RuntimeEnvFileOwner = { write: () => Effect.die("unused"), @@ -2929,6 +2930,7 @@ describe("production runtime", () => { stop: () => Effect.void, remove: () => Effect.void, cleanup: () => Effect.void, + wipePersistentData: () => Effect.void, }; const envOwner: RuntimeEnvFileOwner = { write: () => Effect.die("unused"), diff --git a/packages/stack/src/runtime/schema-init.integration.test.ts b/packages/stack/src/runtime/schema-init.integration.test.ts new file mode 100644 index 0000000000..5ab3ade2e0 --- /dev/null +++ b/packages/stack/src/runtime/schema-init.integration.test.ts @@ -0,0 +1,453 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, FileSystem, Option, Redacted, Stream } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- capture env-file contents before the scoped workspace is removed +import { readFileSync } from "node:fs"; +import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; +import { RequiresActivatedProcessError } from "../public/Errors.ts"; +import { StackIdSchema } from "../public/StackId.ts"; +import type { RuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; +import type { + ContainerContainerSpec, + ContainerEngine, + ContainerNetworkSpec, + ContainerResource, + ContainerVolumeSpec, +} from "./ContainerEngine.ts"; +import { schemaInitContainerName } from "./ContainerRuntime.ts"; +import { schemaInitWorkloads } from "./SchemaInit.ts"; + +interface FakeContainerState { + resources: Array; + calls: Array; + createdSpecs: Array; + envFiles: Map; + nextId: number; +} + +const fakeContainerEngine = (state: FakeContainerState): ContainerEngine => { + const id = (prefix: string): string => `${prefix}-${state.nextId++}`; + const find = (resourceId: string): ContainerResource | undefined => + state.resources.find((resource) => resource.id === resourceId); + return { + kind: "docker", + preflight: Effect.succeed({ host: "host.docker.internal" }), + probe: Effect.void, + inspectImage: () => Effect.succeed({ present: true }), + pullImage: () => Effect.void, + listResources: () => + Effect.sync(() => { + state.calls.push("list-resources"); + return [...state.resources]; + }), + createNetwork: (spec: ContainerNetworkSpec) => + Effect.sync(() => { + state.calls.push("create-network"); + const resource: ContainerResource = { + id: id("network"), + name: spec.name, + kind: "network", + labels: spec.labels, + }; + state.resources.push(resource); + return resource; + }), + removeNetwork: (resourceId: string) => + Effect.sync(() => { + state.calls.push(`remove-network:${resourceId}`); + state.resources = state.resources.filter((resource) => resource.id !== resourceId); + }), + createVolume: (spec: ContainerVolumeSpec) => + Effect.sync(() => { + const resource: ContainerResource = { + id: id("volume"), + name: spec.name, + kind: "volume", + labels: spec.labels, + }; + state.resources.push(resource); + return resource; + }), + removeVolume: (resourceId: string) => + Effect.sync(() => { + state.resources = state.resources.filter((resource) => resource.id !== resourceId); + }), + createContainer: (spec: ContainerContainerSpec) => + Effect.sync(() => { + state.calls.push("create-container"); + if (spec.envFile !== undefined) + state.envFiles.set(spec.envFile, readFileSync(spec.envFile, "utf8")); + state.createdSpecs.push(spec); + const resource: ContainerResource = { + id: id("container"), + name: spec.name, + kind: spec.role, + labels: spec.labels, + state: "created", + }; + state.resources.push(resource); + return resource; + }), + copyToContainer: () => Effect.void, + startContainer: (resourceId: string) => + Effect.sync(() => { + state.calls.push(`start:${resourceId}`); + const resource = find(resourceId); + if (resource !== undefined) + state.resources = state.resources.map((entry) => + entry.id === resourceId ? { ...entry, state: "running" } : entry, + ); + }), + waitContainer: (resourceId: string) => + Effect.sync(() => { + state.calls.push(`wait:${resourceId}`); + return 0; + }), + stopContainer: () => Effect.void, + removeContainer: (resourceId: string) => + Effect.sync(() => { + state.calls.push(`remove:${resourceId}`); + state.resources = state.resources.filter((resource) => resource.id !== resourceId); + }), + streamLogs: () => Stream.empty, + }; +}; + +const fakePreparer: RuntimeArtifactPreparer = { + prepare: (_runtime, workload: PlannedWorkload) => + Effect.succeed({ + workloadId: workload.id, + capability: workload.capability, + version: "1", + outcome: "cached", + artifactRoot: "/tmp/schema-init-artifact", + executablePath: "bin/prepare", + image: `example/${workload.capability}:1`, + }), +}; + +const liveStackId = StackIdSchema.make("b".repeat(64)); +const password = Redacted.make("s3cret"); +const jwtSecret = Redacted.make("jwt-secret-value-that-is-long-enough"); + +const envFromFile = (text: string): Record => + Object.fromEntries( + text + .split("\n") + .filter((line) => line.includes("=")) + .map((line) => { + const index = line.indexOf("="); + return [line.slice(0, index), line.slice(index + 1)]; + }), + ); + +describe("schemaInit", () => { + it.live("runs container one-shots with distinct names and empty publications", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-live-" }); + const state: FakeContainerState = { + resources: [ + { + id: "net-live", + name: "supabase-live-network", + kind: "network", + labels: { + stackId: liveStackId, + ownerSessionId: "owner-session", + role: "network", + }, + }, + ], + calls: [], + createdSpecs: [], + envFiles: new Map(), + nextId: 1, + }; + yield* schemaInitWorkloads( + ["auth"], + { + kind: "live", + stackId: liveStackId, + projectRoot, + runtime: { kind: "container", engine: "docker" }, + config: {}, + databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", + secrets: { databasePassword: password, jwtSecret }, + }, + { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, + ); + expect(state.createdSpecs).toHaveLength(1); + const spec = state.createdSpecs[0]; + expect(spec).toBeDefined(); + if (spec === undefined) return; + expect(spec.name.endsWith("-schema-init")).toBe(true); + expect(spec.publications).toEqual([]); + expect(spec.network).toBe("net-live"); + expect(spec.entrypoint).toBe("/usr/local/bin/auth"); + expect(spec.command).toEqual(["migrate"]); + expect(spec.envFile).toBeDefined(); + if (spec.envFile === undefined) return; + const env = envFromFile(state.envFiles.get(spec.envFile) ?? ""); + expect(env.GOTRUE_DB_DATABASE_URL).toContain("@supabase-database:5432/postgres"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("joins the ephemeral Postgres network and dials supabase-database:5432", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-eph-" }); + const state: FakeContainerState = { + resources: [], + calls: [], + createdSpecs: [], + envFiles: new Map(), + nextId: 1, + }; + yield* schemaInitWorkloads( + ["auth"], + { + kind: "ephemeral", + projectRoot, + runtime: { kind: "container", engine: "docker" }, + config: {}, + databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", + secrets: { databasePassword: password, jwtSecret }, + networkId: "net-eph", + }, + { + containerEngine: fakeContainerEngine(state), + artifactPreparer: fakePreparer, + platform: "linux", + }, + ); + expect(state.calls).not.toContain("create-network"); + const spec = state.createdSpecs[0]; + expect(spec).toBeDefined(); + if (spec === undefined) return; + expect(spec.name).toBe( + schemaInitContainerName({ + stackId: spec.labels.stackId, + workloadId: "auth:auth", + }), + ); + expect(spec.network).toBe("net-eph"); + expect(spec.extraHosts).toEqual(["host.docker.internal:host-gateway"]); + expect(spec.envFile).toBeDefined(); + if (spec.envFile === undefined) return; + const env = envFromFile(state.envFiles.get(spec.envFile) ?? ""); + expect(env.GOTRUE_DB_DATABASE_URL).toContain("@supabase-database:5432/postgres"); + expect(env.GOTRUE_DB_DATABASE_URL).toContain("supabase_auth_admin"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "rewrites ephemeral docker endpoints to the published URL without a cluster network", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "schema-init-eph-rewrite-", + }); + const state: FakeContainerState = { + resources: [], + calls: [], + createdSpecs: [], + envFiles: new Map(), + nextId: 1, + }; + yield* schemaInitWorkloads( + ["auth"], + { + kind: "ephemeral", + projectRoot, + runtime: { kind: "container", engine: "docker" }, + config: {}, + databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", + secrets: { databasePassword: password, jwtSecret }, + }, + { + containerEngine: fakeContainerEngine(state), + artifactPreparer: fakePreparer, + platform: "linux", + }, + ); + expect(state.calls).toContain("create-network"); + const spec = state.createdSpecs[0]; + expect(spec).toBeDefined(); + if (spec === undefined) return; + expect(spec.network).not.toBe("net-eph"); + expect(spec.extraHosts).toEqual(["host.docker.internal:host-gateway"]); + expect(spec.envFile).toBeDefined(); + if (spec.envFile === undefined) return; + const env = envFromFile(state.envFiles.get(spec.envFile) ?? ""); + expect(env.GOTRUE_DB_DATABASE_URL).toContain("@host.docker.internal:54322/postgres"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("resolves pooler env without an activated pooler process", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-pooler-" }); + const state: FakeContainerState = { + resources: [ + { + id: "net-live", + name: "supabase-live-network", + kind: "network", + labels: { + stackId: liveStackId, + ownerSessionId: "owner-session", + role: "network", + }, + }, + ], + calls: [], + createdSpecs: [], + envFiles: new Map(), + nextId: 1, + }; + yield* schemaInitWorkloads( + ["pooler"], + { + kind: "live", + stackId: liveStackId, + projectRoot, + runtime: { kind: "container", engine: "docker" }, + config: {}, + databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", + secrets: { databasePassword: password, jwtSecret }, + }, + { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, + ); + expect(state.createdSpecs).toHaveLength(2); + expect(state.createdSpecs.map((spec) => spec.entrypoint)).toEqual([ + "/app/bin/prepare", + "/app/bin/provision-tenant", + ]); + const envFile = state.createdSpecs[0]?.envFile; + expect(envFile).toBeDefined(); + if (envFile === undefined) return; + const env = envFromFile(state.envFiles.get(envFile) ?? ""); + expect(env.DATABASE_URL).toContain("@supabase-database:5432/_supabase"); + expect(env.POSTGRES_HOST).toBe("supabase-database"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("tags docker analytics as requiring an activated process", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-analytics-" }); + const state: FakeContainerState = { + resources: [], + calls: [], + createdSpecs: [], + envFiles: new Map(), + nextId: 1, + }; + const exit = yield* schemaInitWorkloads( + ["analytics"], + { + kind: "ephemeral", + projectRoot, + runtime: { kind: "container", engine: "docker" }, + config: {}, + databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", + secrets: { databasePassword: password, jwtSecret }, + }, + { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, + ).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (!Exit.isFailure(exit)) return; + const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + expect(error).toBeInstanceOf(RequiresActivatedProcessError); + if (!(error instanceof RequiresActivatedProcessError)) return; + expect(error.capability).toBe("analytics"); + expect(state.createdSpecs).toEqual([]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("skips a disabled capability without creating a one-shot", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-disabled-" }); + const state: FakeContainerState = { + resources: [], + calls: [], + createdSpecs: [], + envFiles: new Map(), + nextId: 1, + }; + yield* schemaInitWorkloads( + ["analytics"], + { + kind: "ephemeral", + projectRoot, + runtime: { kind: "container", engine: "docker" }, + config: { + capabilities: { + analytics: { enabled: false }, + }, + }, + databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", + secrets: { databasePassword: password, jwtSecret }, + }, + { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, + ); + expect(state.createdSpecs).toEqual([]); + expect(state.calls.filter((call) => call === "create-container")).toEqual([]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("schema-inits the trio when Studio is on and analytics is off", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-trio-" }); + const state: FakeContainerState = { + resources: [], + calls: [], + createdSpecs: [], + envFiles: new Map(), + nextId: 1, + }; + yield* schemaInitWorkloads( + ["auth", "storage", "realtime"], + { + kind: "ephemeral", + projectRoot, + runtime: { kind: "container", engine: "docker" }, + config: { + capabilities: { + studio: { enabled: true }, + analytics: { enabled: false }, + }, + }, + databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", + secrets: { databasePassword: password, jwtSecret }, + }, + { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, + ); + const workloads = new Set(state.createdSpecs.map((spec) => spec.labels.workloadId)); + expect(workloads.has("auth:auth")).toBe(true); + expect(workloads.has("storage:storage")).toBe(true); + expect(workloads.has("realtime:realtime")).toBe(true); + expect(workloads.has("studio:studio")).toBe(false); + expect(workloads.has("mail:mail")).toBe(false); + expect(workloads.has("functions:edge-runtime")).toBe(false); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/stack/src/runtime/schema-init.unit.test.ts b/packages/stack/src/runtime/schema-init.unit.test.ts new file mode 100644 index 0000000000..cf79797878 --- /dev/null +++ b/packages/stack/src/runtime/schema-init.unit.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "@effect/vitest"; +import { serializeCommonContainerCommand } from "./ContainerEngine.ts"; +import { StackIdSchema } from "../public/StackId.ts"; +import { + parseSchemaInitDatabaseUrl, + rewriteDatabaseEnvironment, + schemaInitArtifactIdentity, + schemaInitHostGatewayExtraHosts, +} from "./SchemaInit.ts"; + +describe("parseSchemaInitDatabaseUrl", () => { + it("keeps user, database name, host, port, and password", () => { + const parsed = parseSchemaInitDatabaseUrl( + "postgresql://supabase_auth_admin:s3cret%40x@127.0.0.1:54322/_supabase", + ); + expect(parsed).toEqual({ + host: "127.0.0.1", + port: 54322, + password: "s3cret@x", + database: "_supabase", + }); + }); + + it("rejects a non-postgres URL", () => { + expect(parseSchemaInitDatabaseUrl("https://example.test/postgres")).toBeUndefined(); + }); +}); + +describe("rewriteDatabaseEnvironment", () => { + it("rewrites host, port, and password without changing user or database name", () => { + const rewritten = rewriteDatabaseEnvironment( + { + DB_HOST: "supabase-database", + DB_PORT: "5432", + DB_PASSWORD: "old", + GOTRUE_DB_DATABASE_URL: + "postgresql://supabase_auth_admin:old@supabase-database:5432/postgres", + DATABASE_URL: "ecto://supabase_admin:old@supabase-database:5432/_supabase", + API_EXTERNAL_URL: "http://127.0.0.1:54321", + }, + { host: "host.docker.internal", port: 54322, password: "fresh" }, + ); + expect(rewritten.DB_HOST).toBe("host.docker.internal"); + expect(rewritten.DB_PORT).toBe("54322"); + expect(rewritten.DB_PASSWORD).toBe("fresh"); + expect(rewritten.GOTRUE_DB_DATABASE_URL).toBe( + "postgresql://supabase_auth_admin:fresh@host.docker.internal:54322/postgres", + ); + expect(rewritten.DATABASE_URL).toBe( + "ecto://supabase_admin:fresh@host.docker.internal:54322/_supabase", + ); + expect(rewritten.API_EXTERNAL_URL).toBe("http://127.0.0.1:54321"); + }); +}); + +describe("schemaInitHostGatewayExtraHosts", () => { + it("adds host-gateway only on Linux Engine for host.docker.internal", () => { + expect(schemaInitHostGatewayExtraHosts("linux", "host.docker.internal")).toEqual([ + "host.docker.internal:host-gateway", + ]); + expect(schemaInitHostGatewayExtraHosts("darwin", "host.docker.internal")).toEqual([]); + expect(schemaInitHostGatewayExtraHosts("linux", "127.0.0.1")).toEqual([]); + }); +}); + +describe("schemaInitArtifactIdentity", () => { + it("returns a version:image pin and rejects unknown releases", () => { + const identity = schemaInitArtifactIdentity("auth"); + expect(identity).toMatch(/^v.+:/); + expect(schemaInitArtifactIdentity("auth", "not-a-catalog-release")).toBeUndefined(); + }); +}); + +describe("serializeCommonContainerCommand extra hosts", () => { + it("emits --add-host after the network flags", () => { + const request = serializeCommonContainerCommand({ + operation: "create-container", + spec: { + name: "schema-init", + image: "example/auth:1", + labels: { + stackId: StackIdSchema.make("a".repeat(64)), + ownerSessionId: "owner", + workloadId: "auth:auth", + startup: true, + role: "workload", + }, + network: "net", + mounts: [], + volumeMounts: [], + publications: [], + role: "workload", + extraHosts: ["host.docker.internal:host-gateway"], + }, + }); + const networkIndex = request.args.indexOf("--network"); + const addHostIndex = request.args.indexOf("--add-host"); + expect(networkIndex).toBeGreaterThan(-1); + expect(addHostIndex).toBeGreaterThan(networkIndex); + expect(request.args[addHostIndex + 1]).toBe("host.docker.internal:host-gateway"); + }); +}); diff --git a/packages/stack/src/state/PortCoordinator.ts b/packages/stack/src/state/PortCoordinator.ts index a2edfb1e5b..5ec17a7fbf 100644 --- a/packages/stack/src/state/PortCoordinator.ts +++ b/packages/stack/src/state/PortCoordinator.ts @@ -104,21 +104,17 @@ const readAuthoritativeStates = (options: PortCoordinatorOptions) => const ids = entries.filter((entry) => idPattern.test(entry)); const values = yield* Effect.forEach(ids, (id) => options.store.read(id).pipe( - Effect.mapError((error) => - error instanceof StackStateFormatUnsupportedError - ? new StackStateFormatUnsupportedError({ - ...error, - message: `Unable to read sibling stack state ${id}: ${error.message}`, - }) - : error instanceof StackStateInvalidError - ? new StackStateInvalidError({ - ...error, - message: `Unable to read sibling stack state ${id}: ${error.message}`, - path: path.join(root, id, "state.json"), - }) - : error, + Effect.catchIf( + (error): error is StackStateInvalidError | StackStateFormatUnsupportedError => + error instanceof StackStateFormatUnsupportedError || + error instanceof StackStateInvalidError, + (error) => + isMissingStateRemnantError(error) + ? Effect.void + : Effect.logWarning( + `Skipping unreadable sibling stack state ${id}: ${error.message}`, + ).pipe(Effect.as(undefined)), ), - Effect.catchIf(isMissingStateRemnantError, () => Effect.void), Effect.map((state) => (state === undefined ? undefined : { stackId: id, state })), ), ); diff --git a/packages/stack/src/state/ports.integration.test.ts b/packages/stack/src/state/ports.integration.test.ts index 0af0f4c10d..5b8b9fccda 100644 --- a/packages/stack/src/state/ports.integration.test.ts +++ b/packages/stack/src/state/ports.integration.test.ts @@ -19,7 +19,6 @@ import { deriveStackId, type StackIdentity } from "../identity/Identity.ts"; import { PortAllocationError, PortUnavailableError, - StackStateFormatUnsupportedError, StackStateInvalidError, } from "../public/Errors.ts"; import { @@ -96,7 +95,7 @@ const run = (effect: Effect.Effect) => Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); describe("port acquisition", () => { - it.live("requires running state and fails closed on an unreadable sibling", () => + it.live("requires running state and skips an unreadable sibling", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -115,7 +114,26 @@ describe("port acquisition", () => { yield* fs.makeDirectory(siblingRoot, { recursive: true }); yield* fs.writeFileString(path.join(siblingRoot, "state.json"), "not-json"); yield* store.replaceUnlocked(id, { ...state(id, value), desiredLifecycle: "running" }); - const result = yield* coordinator.acquire(id, intents(), []).pipe(Effect.exit); + const result = yield* coordinator.acquire(id, intents(), []); + expect(result.assignments.api?.port).toBeGreaterThan(0); + }), + ), + ); + + it.live("fails closed on this stack's own unreadable state", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-own-corrupt-" }); + const store = yield* makeStackStateStore({ stateRoot: root }); + const value = identity(root, "own-corrupt"); + const id = yield* deriveStackId(value); + yield* store.initialize(id, state(id, value)); + yield* fs.writeFileString(path.join(root, id, "state.json"), "not-json"); + const result = yield* makePortCoordinator(coordinatorOptions(store, root)) + .acquire(id, intents(), []) + .pipe(Effect.exit); expect(Exit.isFailure(result)).toBe(true); if (Exit.isFailure(result)) expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBeInstanceOf( @@ -149,7 +167,7 @@ describe("port acquisition", () => { ), ); - it.live("preserves unsupported sibling format errors with sibling context", () => + it.live("skips an unsupported sibling format and still allocates", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -169,16 +187,45 @@ describe("port acquisition", () => { Schema.fromJsonString(Schema.Unknown), )(siblingState); yield* fs.writeFileString(path.join(root, siblingId, "state.json"), encodedSiblingState); - const result = yield* makePortCoordinator(coordinatorOptions(store, root)) - .acquire(ownId, intents(), []) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (!Exit.isFailure(result)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toBeInstanceOf(StackStateFormatUnsupportedError); - if (!(error instanceof StackStateFormatUnsupportedError)) return; - expect(error.format).toBe("supabase-stack-state-v2"); - expect(error.message).toContain(siblingId); + const result = yield* makePortCoordinator(coordinatorOptions(store, root)).acquire( + ownId, + intents(), + [], + ); + expect(result.assignments.api?.port).toBeGreaterThan(0); + }), + ), + ); + + it.live("skips a legacy sibling identity.checkoutRoot field and still allocates", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-legacy-" }); + const store = yield* makeStackStateStore({ stateRoot: root }); + const ownIdentity = identity(root, "legacy-owner"); + const ownId = yield* deriveStackId(ownIdentity); + const siblingId = yield* deriveStackId(identity(root, "legacy-sibling")); + yield* store.initialize(ownId, state(ownId, ownIdentity)); + const siblingState = { + ...state(siblingId, identity(root, "legacy-sibling")), + identity: { + ...identity(root, "legacy-sibling"), + checkoutRoot: root, + }, + }; + yield* fs.makeDirectory(path.join(root, siblingId), { recursive: true }); + const encodedSiblingState = yield* Schema.encodeEffect( + Schema.fromJsonString(Schema.Unknown), + )(siblingState); + yield* fs.writeFileString(path.join(root, siblingId, "state.json"), encodedSiblingState); + const result = yield* makePortCoordinator(coordinatorOptions(store, root)).acquire( + ownId, + intents(), + [], + ); + expect(result.assignments.api?.port).toBeGreaterThan(0); }), ), ); diff --git a/packages/stack/src/supervisor/SessionLauncher.ts b/packages/stack/src/supervisor/SessionLauncher.ts index 8f996d9309..450daf7de2 100644 --- a/packages/stack/src/supervisor/SessionLauncher.ts +++ b/packages/stack/src/supervisor/SessionLauncher.ts @@ -17,6 +17,8 @@ export interface SessionLauncher { readonly launch: (plan: ExecutionPlan) => Effect.Effect; /** Stops and removes every workload started in this session in reverse order. */ readonly stop: Effect.Effect; + /** Drops session ownership after the caller already stopped those workloads. */ + readonly forget: (workloadIds: ReadonlyArray) => Effect.Effect; /** Whether the most recent launch/rollback cleanup completed exactly. */ readonly cleanupProven: Effect.Effect; /** Clears the session after stack-wide runtime cleanup has completed. */ @@ -190,9 +192,14 @@ export const makeSessionLauncher = (options: { }); const stop = Effect.suspend(() => Ref.get(session).pipe(Effect.flatMap(cleanup))); + const forget = (workloadIds: ReadonlyArray) => + Ref.update(session, (current) => + current.filter((entry) => !workloadIds.includes(entry.key.workloadId)), + ); return { launch, stop, + forget, cleanupProven: Ref.get(cleanupProven), clear: Ref.set(session, []), } satisfies SessionLauncher; diff --git a/packages/stack/src/supervisor/Supervisor.ts b/packages/stack/src/supervisor/Supervisor.ts index 4038bae7a3..73283b7b0d 100644 --- a/packages/stack/src/supervisor/Supervisor.ts +++ b/packages/stack/src/supervisor/Supervisor.ts @@ -72,6 +72,14 @@ import { import type { ActivationResult } from "../gateway/Gateway.ts"; +const RESET_DATABASE_BOUNCE_CAPABILITIES: ReadonlySet = new Set([ + "auth", + "storage", + "realtime", + "pooler", + "analytics", +]); + interface SupervisorLaunchAttempt { /** Rolls back only workloads and ingress acquired by this launch. */ readonly rollback: Effect.Effect; @@ -105,6 +113,8 @@ export interface Supervisor { readonly config?: StackConfig; }) => Effect.Effect; readonly destroy: Effect.Effect; + /** Wipes Postgres data for the running stack and bootstraps a fresh cluster. */ + readonly resetDatabase: Effect.Effect; /** Completes after a successful stop or destroy shutdown signal. */ readonly shutdown: Effect.Effect; /** Shuts down only when durable state is absent or cleanly non-running. */ @@ -274,7 +284,7 @@ export const makeSupervisor = ( }); const joinExit = (result: Exit.Exit): Effect.Effect => Exit.isSuccess(result) ? Effect.succeed(result.value) : Effect.failCause(result.cause); - type LifecycleKind = "start" | "stop" | "destroy"; + type LifecycleKind = "start" | "stop" | "destroy" | "reset"; type LifecycleResult = Deferred.Deferred, never>; type ActiveLifecycle = Readonly<{ kind: LifecycleKind; @@ -289,7 +299,7 @@ export const makeSupervisor = ( // installing its workloads. Wait for that shared lifecycle result before attempting lazy // activation; otherwise the phase check below would turn a valid cold request into 503. const lifecycle = yield* Ref.get(lifecycleActive); - if (lifecycle?.kind === "start") { + if (lifecycle?.kind === "start" || lifecycle?.kind === "reset") { const started = yield* Deferred.await(lifecycle.result); yield* joinExit(started); } @@ -676,6 +686,63 @@ export const makeSupervisor = ( yield* submitLifecycle("start", startOperation(startOptions)); return yield* snapshot(); }); + const resetDatabaseOperation = () => + Effect.gen(function* () { + const previous = yield* Ref.get(phase); + if (previous !== "running") + return yield* new StackNotRunningError({ + stackId: options.stackId, + message: "Stack is not running", + }); + const state = yield* read(); + if (state === undefined || state.definition === undefined) + return yield* new StackStateInvalidError({ message: "Stack state is missing" }); + const status = yield* snapshot(); + const database = status.capabilities.find((capability) => capability.name === "database"); + if (database?.state !== "ready") + return yield* new StackNotRunningError({ + stackId: options.stackId, + message: "Database is not running", + }); + const plan = yield* rebuildExecutionPlan(state.runtime, state.definition).pipe( + Effect.provideContext(options.context), + Effect.mapError( + (error) => new StackStateInvalidError({ message: error.message, cause: error }), + ), + ); + const bounceNames = new Set( + status.capabilities.flatMap((capability) => + capability.state === "ready" && RESET_DATABASE_BOUNCE_CAPABILITIES.has(capability.name) + ? [capability.name] + : [], + ), + ); + const bounce = plan.workloads.filter((workload) => bounceNames.has(workload.capability)); + const databaseWorkload = plan.workloads.find( + (workload) => workload.id === "database:database", + ); + if (databaseWorkload === undefined) + return yield* new StackStateInvalidError({ message: "Database workload is missing" }); + const stopOne = (workloadId: string) => + Effect.gen(function* () { + const key = { stackId: options.stackId, workloadId: workloadId }; + yield* runtime.driver.stop(key).pipe(Effect.mapError(mapRuntimeError)); + yield* runtime.driver.remove(key).pipe(Effect.mapError(mapRuntimeError)); + yield* launcher.forget([workloadId]); + }); + for (const workload of [...bounce].reverse()) yield* stopOne(workload.id); + yield* stopOne(databaseWorkload.id); + yield* runtime.driver + .wipePersistentData({ stackId: options.stackId, workloadId: databaseWorkload.id }) + .pipe(Effect.mapError(mapRuntimeError)); + const resetWorkloads = [databaseWorkload, ...bounce]; + yield* launcher + .launch({ ...plan, workloads: resetWorkloads }) + .pipe(Effect.mapError(mapRuntimeError)); + }); + const resetDatabase = submitLifecycle("reset", resetDatabaseOperation()).pipe( + Effect.andThen(snapshot()), + ); const stopOperation = () => Effect.gen(function* () { const previous = yield* Ref.get(phase); @@ -821,12 +888,6 @@ export const makeSupervisor = ( ), ); - const auth = definition.capabilities.auth; - if (!auth.enabled) - return yield* Effect.fail( - rpcError("InvalidStackConfigError", "Stack credentials require Auth to be enabled"), - ); - const requiredSecret = (slot: string): Effect.Effect => { const value = state.secrets[slot]?.value; return value === undefined || value.length === 0 @@ -842,22 +903,22 @@ export const makeSupervisor = ( databasePassword, )}@${databaseHost}:${databaseAssignment.port}/postgres`; - const publishableKey = yield* requiredSecret(AUTH_PUBLISHABLE_KEY_SLOT); - const secretKey = yield* requiredSecret(AUTH_SECRET_KEY_SLOT); - const anonJwt = yield* requiredSecret(AUTH_ANON_KEY_SLOT); - const serviceRoleJwt = yield* requiredSecret(AUTH_SERVICE_ROLE_KEY_SLOT); + const auth = definition.capabilities.auth; + const api = auth.enabled + ? { + publishableKey: yield* requiredSecret(AUTH_PUBLISHABLE_KEY_SLOT), + secretKey: Redacted.make(yield* requiredSecret(AUTH_SECRET_KEY_SLOT)), + anonJwt: yield* requiredSecret(AUTH_ANON_KEY_SLOT), + serviceRoleJwt: Redacted.make(yield* requiredSecret(AUTH_SERVICE_ROLE_KEY_SLOT)), + } + : undefined; const base: EffectStackCredentials = { database: { url: Redacted.make(databaseUrl), password: Redacted.make(databasePassword), }, - api: { - publishableKey, - secretKey: Redacted.make(secretKey), - anonJwt, - serviceRoleJwt: Redacted.make(serviceRoleJwt), - }, + ...(api === undefined ? {} : { api }), }; const storage = definition.capabilities.storage; const s3 = storage.settings.s3_protocol; @@ -904,11 +965,13 @@ export const makeSupervisor = ( credentials: () => credentials, start: ({ config }: { readonly config?: StackConfig }) => operation(start({ config })), destroy: () => operation(destroy), + resetDatabase: () => operation(resetDatabase), logs: (query: LogQuery) => operation(logs(query)), }); return { status, start, + resetDatabase, destroy, shutdown: Deferred.await(shutdownSignal), shutdownIfIdle, diff --git a/packages/stack/src/supervisor/handles.integration.test.ts b/packages/stack/src/supervisor/handles.integration.test.ts index b7e4ada27d..a52836a6ef 100644 --- a/packages/stack/src/supervisor/handles.integration.test.ts +++ b/packages/stack/src/supervisor/handles.integration.test.ts @@ -500,6 +500,7 @@ describe("managed stack handles", { timeout: 30_000 }, () => { start: () => Effect.fail({ tag: "StackPreparationError", message: "artifact is incomplete" }), destroy: () => Effect.void, + resetDatabase: () => Effect.succeed(status), logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), }; yield* startControlServer({ @@ -604,6 +605,7 @@ describe("managed stack handles", { timeout: 30_000 }, () => { Effect.andThen(Deferred.await(responseRelease)), Effect.asVoid, ), + resetDatabase: () => Effect.succeed(status), logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), }, onShutdownReady: Deferred.succeed(callbackStarted, undefined).pipe( diff --git a/packages/stack/src/supervisor/session-launcher.integration.test.ts b/packages/stack/src/supervisor/session-launcher.integration.test.ts index 45b6e68a59..0a79852e84 100644 --- a/packages/stack/src/supervisor/session-launcher.integration.test.ts +++ b/packages/stack/src/supervisor/session-launcher.integration.test.ts @@ -92,6 +92,7 @@ describe("session launcher", () => { stop: (key) => Effect.sync(() => calls.push(`stop:${key.workloadId}`)), remove: (key) => Effect.sync(() => calls.push(`remove:${key.workloadId}`)), cleanup: () => Effect.void, + wipePersistentData: () => Effect.void, }; const launcher = yield* makeSessionLauncher({ stackId, driver }); const launching = yield* Effect.forkChild(launcher.launch(plan([database, mail, rest])), { @@ -134,6 +135,7 @@ describe("session launcher", () => { stop: () => Effect.void, remove: () => Effect.void, cleanup: () => Effect.void, + wipePersistentData: () => Effect.void, }; const launcher = yield* makeSessionLauncher({ stackId, driver }); const launching = yield* Effect.forkChild(launcher.launch(plan([database, mail, rest])), { @@ -197,6 +199,7 @@ describe("session launcher", () => { stop: () => Effect.die("unreachable"), remove: () => Effect.die("unreachable"), cleanup: () => Effect.void, + wipePersistentData: () => Effect.void, }; const launcher = yield* makeSessionLauncher({ stackId, driver }); const result = yield* launcher diff --git a/packages/stack/src/supervisor/startup-ingress.integration.test.ts b/packages/stack/src/supervisor/startup-ingress.integration.test.ts index 303446a565..d3e8d44860 100644 --- a/packages/stack/src/supervisor/startup-ingress.integration.test.ts +++ b/packages/stack/src/supervisor/startup-ingress.integration.test.ts @@ -145,6 +145,7 @@ const makeStartupFixture = () => stop: () => Effect.void, remove: () => Effect.void, cleanup: () => Effect.void, + wipePersistentData: () => Effect.void, }; const entry: StackLogEntry = { cursor: { opaque: "v1_1" }, diff --git a/packages/stack/src/supervisor/supervisor.integration.test.ts b/packages/stack/src/supervisor/supervisor.integration.test.ts index 49921c7d8e..bc7bdc15bc 100644 --- a/packages/stack/src/supervisor/supervisor.integration.test.ts +++ b/packages/stack/src/supervisor/supervisor.integration.test.ts @@ -96,6 +96,7 @@ const makeFixture = ( readonly stopStarted?: Deferred.Deferred; readonly workloadStopFailFirst?: Ref.Ref; readonly workloadRemoveFailFirst?: Ref.Ref; + readonly wipeFailFirst?: Ref.Ref; readonly stopFailFirst?: Ref.Ref; readonly destroyGate?: Deferred.Deferred; readonly destroyStarted?: Deferred.Deferred; @@ -328,6 +329,26 @@ const makeFixture = ( if (!destroy && gateStopCleanup) yield* Ref.update(logEntries, (current) => [...current, finalEntry]); }), + wipePersistentData: (key) => + Effect.gen(function* () { + if (fixtureOptions.wipeFailFirst !== undefined) { + const fail = yield* Ref.get(fixtureOptions.wipeFailFirst); + if (fail) { + yield* Ref.set(fixtureOptions.wipeFailFirst, false); + return yield* new RuntimeDriverError({ + message: "injected wipe failure", + stackId: key.stackId, + workloadId: key.workloadId, + }); + } + } + if (fixtureOptions.timeline !== undefined) + yield* Ref.update(fixtureOptions.timeline, (current) => [ + ...current, + `wipe:${key.workloadId}`, + ]); + yield* Ref.update(calls, (current) => [...current, `wipe:${key.workloadId}`]); + }), }; const runtime: SupervisorRuntime = { driver, @@ -1229,6 +1250,9 @@ describe("Supervisor composition", () => { /^postgresql:\/\/postgres:.+@127\.0\.0\.1:\d+\/postgres$/, ); expect(Redacted.value(credentials.database.password)).toEqual(expect.any(String)); + expect(credentials.api).toBeDefined(); + if (credentials.api === undefined) + return yield* new StackStateInvalidError({ message: "API credentials are missing" }); expect(credentials.api.publishableKey).toEqual(expect.any(String)); expect(Redacted.value(credentials.api.secretKey)).toEqual(expect.any(String)); expect(credentials.api.anonJwt).toEqual(expect.any(String)); @@ -1320,12 +1344,15 @@ describe("Supervisor composition", () => { ), ); - it.live("fails closed when Auth is disabled", () => + it.live("returns database credentials when Auth is disabled", () => run( Effect.gen(function* () { const { fixture } = yield* makeCredentialsFixture({ authEnabled: false }); - const failed = yield* invokeCredentials(fixture.supervisor).pipe(Effect.exit); - expect(errorOf(failed)).toMatchObject({ tag: "InvalidStackConfigError" }); + const credentials = yield* invokeCredentials(fixture.supervisor); + expect(Redacted.value(credentials.database.url)).toMatch( + /^postgresql:\/\/postgres:.+@127\.0\.0\.1:\d+\/postgres$/, + ); + expect(credentials.api).toBeUndefined(); }), ), ); @@ -1492,6 +1519,67 @@ describe("Supervisor composition", () => { ), ); + it.live("wipes only the database workload and bounces ready dependents", () => + run( + Effect.gen(function* () { + const timeline = yield* Ref.make>([]); + const fixture = yield* makeFixture({ timeline }); + yield* fixture.supervisor.start({ + config: { capabilities: { auth: { activation: "eager" } } }, + }); + yield* Ref.set(timeline, []); + yield* Ref.set(fixture.calls, []); + + const status = yield* fixture.supervisor.resetDatabase; + expect(status.lifecycle).toBe("running"); + expect( + status.capabilities.find((capability) => capability.name === "database")?.state, + ).toBe("ready"); + expect(yield* Ref.get(timeline)).toEqual([ + "stop:auth:auth", + "stop:database:database", + "wipe:database:database", + "start:database:database", + "start:auth:auth", + ]); + expect(yield* Ref.get(fixture.calls)).toContain("wipe:database:database"); + }), + ), + ); + + it.live( + "relaunches the database after a wipe failure because stopped workloads were forgotten", + () => + run( + Effect.gen(function* () { + const timeline = yield* Ref.make>([]); + const wipeFailFirst = yield* Ref.make(true); + const fixture = yield* makeFixture({ timeline, wipeFailFirst }); + yield* fixture.supervisor.start(); + yield* Ref.set(timeline, []); + + const resetExit = yield* fixture.supervisor.resetDatabase.pipe(Effect.exit); + expect(Exit.isFailure(resetExit)).toBe(true); + + yield* fixture.supervisor.start(); + expect(yield* Ref.get(timeline)).toEqual([ + "stop:database:database", + "start:database:database", + ]); + }), + ), + ); + + it.live("refuses reset when the database is not running", () => + run( + Effect.gen(function* () { + const fixture = yield* makeFixture(); + const exit = yield* fixture.supervisor.resetDatabase.pipe(Effect.exit); + expect(errorOf(exit)).toBeInstanceOf(StackNotRunningError); + }), + ), + ); + it.live("stops the launched session in reverse dependency order", () => run( Effect.gen(function* () {