diff --git a/.changeset/marketplace-rehydrate-seed-heal.md b/.changeset/marketplace-rehydrate-seed-heal.md new file mode 100644 index 0000000000..6d3594eb2c --- /dev/null +++ b/.changeset/marketplace-rehydrate-seed-heal.md @@ -0,0 +1,26 @@ +--- +"@objectstack/cloud-connection": patch +--- + +fix(marketplace): heal missing sample data when rehydrating installed packages onto a new database + +The install ledger (`.objectstack/installed-packages/`) is anchored to the +project directory while the database can be swapped out from under it — +`os dev --fresh`, a deleted `dev.db`, a `--database` switch. Rehydrate +deliberately never re-seeds (existing rows must not be re-upserted over user +edits on every boot), which left a rehydrated marketplace package PERMANENTLY +empty on a new database: app in the switcher, tables created, zero rows — the +"HotCRM installed but every KPI is 0 / Sales Pipeline all-empty" state. + +Rehydrate now runs the bundled seed datasets iff the manifest actually bundles +them, the user never explicitly purged them, the runtime is single-tenant +(multi-tenant seeding stays owned by the per-org replay), and EVERY seeded +object is empty — one surviving row anywhere means the data is still there and +nothing is touched, so the heal is idempotent across restarts and can never +revert user edits. + +Also fixed along the way: a purge now stamps `sampleDataPurged` on the ledger +entry (so healed restarts respect the deliberate empty baseline), and install +marks `withSampleData: true` when the seed run reports all rows *skipped* +(already present, e.g. a reinstall over live demo data) instead of leaving the +flag false over a seeded database. diff --git a/packages/cloud-connection/src/local-manifest-source.ts b/packages/cloud-connection/src/local-manifest-source.ts index aa51a3be96..f59b96ea7f 100644 --- a/packages/cloud-connection/src/local-manifest-source.ts +++ b/packages/cloud-connection/src/local-manifest-source.ts @@ -44,6 +44,12 @@ export interface InstalledManifestEntry { * database. True after install (seedNow=true) or an explicit reseed; * false after a purge. Persisted so the UI can show "Add" vs "Re-seed". */ withSampleData?: boolean; + /** True only after an explicit purge-sample-data call. The rehydrate-time + * sample-data healer (see MarketplaceInstallLocalPlugin.maybeHealSampleData) + * must not resurrect demo rows the user deliberately removed — an empty + * table after a purge is desired state, not data loss. Cleared again by + * install/reseed runs that land rows. */ + sampleDataPurged?: boolean; } /** Default ledger location, relative to the runtime's working directory. */ diff --git a/packages/cloud-connection/src/marketplace-install-local-heal.test.ts b/packages/cloud-connection/src/marketplace-install-local-heal.test.ts new file mode 100644 index 0000000000..3781dddf97 --- /dev/null +++ b/packages/cloud-connection/src/marketplace-install-local-heal.test.ts @@ -0,0 +1,236 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Rehydrate-time sample-data healing. + * + * The install ledger lives under `/.objectstack/installed-packages/` while + * the database can be swapped out from under it (`os dev --fresh`, a deleted + * dev.db, a `--database` switch). Rehydrate deliberately skips seeding + * (existing rows must not be re-upserted over user edits on every boot), which + * used to leave a rehydrated package PERMANENTLY empty on a new database: app + * in the switcher, tables created, zero rows — the "HotCRM installed but every + * KPI is 0" bug. These tests pin the healer that closes the gap: + * + * • all seeded objects empty → seeds run, `withSampleData` flips true + * • any surviving row → untouched (no silent demo-data reverts) + * • explicit purge → stays empty across restarts + * • multi-tenant mode → left to the per-org replay + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +// What the (mocked) seed loader reports back, plus a call journal so tests can +// assert whether/what the healer actually seeded. +let seedResult: any = { summary: { totalInserted: 0, totalUpdated: 0, totalSkipped: 0 }, errors: [] }; +let loadCalls: any[] = []; + +vi.mock('@objectstack/runtime', () => ({ + SeedLoaderService: class { + async load(request: any) { loadCalls.push(request); return seedResult; } + }, +})); +vi.mock('@objectstack/spec/data', () => ({ + SeedLoaderRequestSchema: { parse: (x: any) => x }, +})); + +import { MarketplaceInstallLocalPlugin } from './marketplace-install-local-plugin.js'; +import { LocalManifestSource } from './local-manifest-source.js'; + +type Handler = (c: any) => Promise; + +function makeRawApp() { + const routes = new Map(); + return { + routes, + get: (p: string, h: Handler) => routes.set(`GET ${p}`, h), + post: (p: string, h: Handler) => routes.set(`POST ${p}`, h), + delete: (p: string, h: Handler) => routes.set(`DELETE ${p}`, h), + }; +} + +function makeCtx(rawApp: any, services: Record) { + const hooks = new Map(); + return { + ctx: { + hook: (e: string, h: any) => hooks.set(e, h), + getService: (name: string) => { + if (name === 'http-server') return { getRawApp: () => rawApp }; + const svc = services[name]; + if (svc === undefined) throw new Error(`no ${name}`); + return svc; + }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }, + fire: async () => { await hooks.get('kernel:ready')?.(); }, + }; +} + +/** A Hono-ish context. `param` carries the :manifestId route value. */ +function makeC(body: any, manifestId?: string) { + const json = vi.fn((payload: any, status?: number) => ({ payload, status: status ?? 200 })); + return { + req: { + url: 'http://localhost:3000/api/v1/marketplace/install-local', + raw: new Request('http://localhost:3000/x'), + json: async () => body, + param: (k: string) => (k === 'manifestId' ? manifestId : undefined), + header: () => undefined, + }, + json, + }; +} + +const MANIFEST = { + id: 'app.test.crm', + version: '1.0.0', + objects: [{ name: 'crm_x', fields: { name: { type: 'text' } } }], + data: [ + { object: 'crm_x', records: [{ id: 'a', name: 'a' }, { id: 'b', name: 'b' }] }, + { object: 'crm_y', records: [{ id: 'c', name: 'c' }] }, + ], +}; + +/** Services with a controllable emptiness probe. */ +function makeServices(findRows: Record) { + return { + manifest: { register: vi.fn() }, + auth: { api: { getSession: async () => ({ user: { id: 'admin' } }) } }, + objectql: { + syncSchemas: async () => undefined, + find: vi.fn(async (object: string) => findRows[object] ?? []), + }, + metadata: {}, + driver: { delete: vi.fn(async () => true) }, + }; +} + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'mil-heal-')); + seedResult = { summary: { totalInserted: 0, totalUpdated: 0, totalSkipped: 0 }, errors: [] }; + loadCalls = []; +}); +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + delete process.env.OS_MULTI_ORG_ENABLED; + vi.restoreAllMocks(); +}); + +/** Pre-write a ledger entry, then boot a fresh plugin so rehydrate runs. */ +async function rehydrateWith(entryOverrides: Record, findRows: Record) { + new LocalManifestSource(dir).write({ + packageId: 'pkg_1', + versionId: 'pkgv_1', + manifestId: MANIFEST.id, + version: MANIFEST.version, + manifest: MANIFEST, + installedAt: '2026-01-01T00:00:00.000Z', + installedBy: 'admin', + withSampleData: false, + ...entryOverrides, + }); + const rawApp = makeRawApp(); + const services = makeServices(findRows); + const { ctx, fire } = makeCtx(rawApp, services); + const plugin = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + await plugin.start(ctx as any); + await fire(); + return { rawApp, services, ctx }; +} + +describe('rehydrate sample-data healing', () => { + it('reseeds a rehydrated package when every seeded object is empty', async () => { + seedResult = { summary: { totalInserted: 3, totalUpdated: 0, totalSkipped: 0 }, errors: [] }; + await rehydrateWith({}, { crm_x: [], crm_y: [] }); + + expect(loadCalls).toHaveLength(1); + // The full bundled datasets are replayed, as an upsert. + expect(loadCalls[0].seeds).toHaveLength(2); + expect(loadCalls[0].config).toMatchObject({ defaultMode: 'upsert', multiPass: true }); + // No org id in single-tenant mode. + expect(loadCalls[0].config.organizationId).toBeUndefined(); + + // The ledger now records that sample data is present again. + const entry = new LocalManifestSource(dir).read(MANIFEST.id)!; + expect(entry.withSampleData).toBe(true); + expect(entry.sampleDataPurged).toBe(false); + }); + + it('does NOT reseed when any seeded object still has rows', async () => { + await rehydrateWith({}, { crm_x: [], crm_y: [{ id: 'c' }] }); + expect(loadCalls).toHaveLength(0); + }); + + it('does NOT resurrect explicitly purged sample data', async () => { + await rehydrateWith({ sampleDataPurged: true }, { crm_x: [], crm_y: [] }); + expect(loadCalls).toHaveLength(0); + }); + + it('leaves multi-tenant seeding to the per-org replay', async () => { + process.env.OS_MULTI_ORG_ENABLED = 'true'; + await rehydrateWith({}, { crm_x: [], crm_y: [] }); + expect(loadCalls).toHaveLength(0); + }); + + it('keeps withSampleData false when the heal run lands no rows', async () => { + seedResult = { summary: { totalInserted: 0, totalUpdated: 0, totalSkipped: 0 }, errors: [{ message: 'database is locked' }] }; + const { ctx } = await rehydrateWith({}, { crm_x: [], crm_y: [] }); + expect(loadCalls).toHaveLength(1); + const entry = new LocalManifestSource(dir).read(MANIFEST.id)!; + expect(entry.withSampleData).toBe(false); + // The failure is loud, with the underlying reason. + expect((ctx.logger.warn as any).mock.calls.some((c: any[]) => String(c[0]).includes('database is locked'))).toBe(true); + }); + + it('purge → restart keeps the package empty (end to end through the endpoints)', async () => { + // Install over an empty DB, with rows landing. + seedResult = { summary: { totalInserted: 3, totalUpdated: 0, totalSkipped: 0 }, errors: [] }; + const rawApp = makeRawApp(); + const services = makeServices({ crm_x: [], crm_y: [] }); + const { ctx, fire } = makeCtx(rawApp, services); + const plugin = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + await plugin.start(ctx as any); + await fire(); + const installRes = await rawApp.routes.get('POST /api/v1/marketplace/install-local')!( + makeC({ manifest: MANIFEST }), + ); + expect(installRes.payload?.success).toBe(true); + + // Purge the sample data. + const purgeRes = await rawApp.routes.get('POST /api/v1/marketplace/install-local/:manifestId/purge-sample-data')!( + makeC({}, MANIFEST.id), + ); + expect(purgeRes.payload?.success).toBe(true); + expect(new LocalManifestSource(dir).read(MANIFEST.id)?.sampleDataPurged).toBe(true); + + // Restart (fresh plugin over the same ledger, DB now empty): no reseed. + loadCalls = []; + const rawApp2 = makeRawApp(); + const { ctx: ctx2, fire: fire2 } = makeCtx(rawApp2, makeServices({ crm_x: [], crm_y: [] })); + const plugin2 = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + await plugin2.start(ctx2 as any); + await fire2(); + expect(loadCalls).toHaveLength(0); + }); + + it('install marks withSampleData=true when rows were skipped (already present)', async () => { + // Reinstall over a database that already holds the demo rows: the + // loader reports all-skipped. The ledger must still say the install + // carries sample data — this was the stale-`false` quirk that made + // older ledgers look purge-like. + seedResult = { summary: { totalInserted: 0, totalUpdated: 0, totalSkipped: 3 }, errors: [] }; + const rawApp = makeRawApp(); + const { ctx, fire } = makeCtx(rawApp, makeServices({ crm_x: [{ id: 'a' }], crm_y: [{ id: 'c' }] })); + const plugin = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + await plugin.start(ctx as any); + await fire(); + const installRes = await rawApp.routes.get('POST /api/v1/marketplace/install-local')!( + makeC({ manifest: MANIFEST }), + ); + expect(installRes.payload?.success).toBe(true); + expect(new LocalManifestSource(dir).read(MANIFEST.id)?.withSampleData).toBe(true); + }); +}); diff --git a/packages/cloud-connection/src/marketplace-install-local-plugin.ts b/packages/cloud-connection/src/marketplace-install-local-plugin.ts index bf53f95085..e5930f45b9 100644 --- a/packages/cloud-connection/src/marketplace-install-local-plugin.ts +++ b/packages/cloud-connection/src/marketplace-install-local-plugin.ts @@ -186,6 +186,14 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // the original install, and multi-tenant orgs will replay // via the security middleware on next sys_organization insert. await this.applySideEffects(ctx, entry.manifest, { seedNow: false }); + // …EXCEPT when the database no longer has them: the ledger is + // anchored to while the database can be swapped out from + // under it (`os dev --fresh`, a deleted dev.db, a --database + // switch). Config-declared apps re-seed on every single-tenant + // boot via AppPlugin, but a rehydrated marketplace package + // would stay empty forever — app visible, tables created, + // zero rows. Heal that case. + await this.maybeHealSampleData(ctx, entry); ctx.logger?.info?.(`[MarketplaceInstallLocal] rehydrated ${entry.manifestId}@${entry.version}`); } catch (err: any) { ctx.logger?.error?.(`[MarketplaceInstallLocal] rehydrate failed for ${entry.manifestId}`, err instanceof Error ? err : new Error(String(err))); @@ -193,6 +201,62 @@ export class MarketplaceInstallLocalPlugin implements Plugin { } }; + /** + * Rehydrate-time sample-data healing (the "installed app, empty database" + * repair). Runs the bundled seed datasets iff: + * + * • the cached manifest actually bundles seed datasets, AND + * • the user never explicitly purged them (`sampleDataPurged`), AND + * • single-tenant mode (multi-tenant seeding is owned by the per-org + * replay on sys_organization insert — the datasets were already merged + * into the kernel's `seed-datasets` service by applySideEffects), AND + * • EVERY seeded object is empty. One surviving row anywhere means the + * install-time rows (or the user's own data) are still present, and + * re-upserting would silently revert user edits on every boot. + * + * Never throws — a failed heal logs and leaves the boot untouched. + */ + private maybeHealSampleData = async (ctx: PluginContext, entry: InstalledEntry): Promise => { + const datasets = Array.isArray(entry.manifest?.data) + ? entry.manifest.data.filter((d: any) => d && d.object && Array.isArray(d.records)) + : []; + if (datasets.length === 0) return; + if (entry.sampleDataPurged === true) return; + if (resolveMultiOrgEnabled()) { + ctx.logger?.info?.(`[MarketplaceInstallLocal] multi-tenant — sample-data heal for ${entry.manifestId} left to per-org replay`); + return; + } + + let ql: any; + try { ql = ctx.getService('objectql'); } catch { return; } + if (!ql || typeof ql.find !== 'function') return; + + // Emptiness probe: any row in any seeded object → nothing to heal. + const objects = [...new Set(datasets.map((d: any) => String(d.object)))]; + for (const object of objects) { + try { + const rows = await ql.find(object, { limit: 1, context: { isSystem: true } } as any); + const first = Array.isArray(rows) ? rows[0] : rows?.items?.[0]; + if (first !== undefined && first !== null) return; + } catch { /* unknown/missing table reads as empty — keep probing */ } + } + + try { + const summary = await this.runInlineSeed(ctx, datasets); + const landed = (summary.inserted ?? 0) + (summary.updated ?? 0); + if (landed > 0) { + entry.withSampleData = true; + entry.sampleDataPurged = false; + try { this.ledger.write(entry); } catch { /* non-fatal */ } + ctx.logger?.info?.(`[MarketplaceInstallLocal] healed sample data for ${entry.manifestId}: inserted=${summary.inserted} updated=${summary.updated} errors=${summary.errors}`); + } else { + ctx.logger?.warn?.(`[MarketplaceInstallLocal] sample-data heal for ${entry.manifestId} landed no rows${summary.errorSample ? ` — first error: ${summary.errorSample}` : ''}`); + } + } catch (err: any) { + ctx.logger?.warn?.(`[MarketplaceInstallLocal] sample-data heal failed for ${entry.manifestId}: ${err?.message ?? err}`); + } + }; + private handleInstall = async (c: any, ctx: PluginContext): Promise => { const userId = await this.requireAuthenticatedUser(c, ctx); if (!userId) { @@ -394,8 +458,14 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // • stash seed datasets on the kernel + run them now so the // installed app has demo data on first paint. const seededSummary = await this.applySideEffects(ctx, manifest, { seedNow: true, c }); - if (seededSummary.seeded.mode === 'inline' && (seededSummary.seeded.inserted ?? 0) + (seededSummary.seeded.updated ?? 0) > 0) { + // `skipped` counts too: an all-skip run means the rows are ALREADY in + // the database (e.g. a reinstall/upgrade over live sample data) — the + // install carries sample data either way. Leaving the flag false here + // is what made older ledgers claim "no sample data" over a seeded DB. + const seededRows = (seededSummary.seeded.inserted ?? 0) + (seededSummary.seeded.updated ?? 0) + (seededSummary.seeded.skipped ?? 0); + if (seededSummary.seeded.mode === 'inline' && seededRows > 0) { entry.withSampleData = true; + entry.sampleDataPurged = false; try { this.ledger.write(entry); } catch { /* non-fatal — entry already on disk */ } @@ -589,6 +659,7 @@ export class MarketplaceInstallLocalPlugin implements Plugin { // Only mark the install as carrying sample data once rows actually landed. try { entry.withSampleData = true; + entry.sampleDataPurged = false; this.ledger.write(entry); } catch { /* non-fatal */ } @@ -671,9 +742,12 @@ export class MarketplaceInstallLocalPlugin implements Plugin { } } - // Flip flag so UI reflects the empty baseline + // Flip flag so UI reflects the empty baseline. `sampleDataPurged` + // additionally tells the rehydrate-time healer this emptiness is + // deliberate — demo rows must not come back on the next restart. try { entry.withSampleData = false; + entry.sampleDataPurged = true; this.ledger.write(entry); } catch { /* non-fatal */ } @@ -713,7 +787,7 @@ export class MarketplaceInstallLocalPlugin implements Plugin { ctx: PluginContext, manifest: any, opts: { seedNow: boolean; c?: any }, - ): Promise<{ translationsLoaded: number; seeded: { mode: 'inline' | 'replayer' | 'skipped'; inserted?: number; updated?: number; errors?: number; reason?: string; errorSample?: string } }> => { + ): Promise<{ translationsLoaded: number; seeded: { mode: 'inline' | 'replayer' | 'skipped'; inserted?: number; updated?: number; skipped?: number; errors?: number; reason?: string; errorSample?: string } }> => { const appId = String(manifest?.id ?? 'unknown'); let translationsLoaded = 0; let seedSummary: any = { mode: 'skipped', reason: 'no-datasets' }; @@ -807,33 +881,9 @@ export class MarketplaceInstallLocalPlugin implements Plugin { } } if (!multiTenant || organizationId) { - const [{ SeedLoaderService }, { SeedLoaderRequestSchema }] = await Promise.all([ - import('@objectstack/runtime'), - import('@objectstack/spec/data'), - ]); - const seedLoader = new (SeedLoaderService as any)(ql, metadata, ctx.logger); - const request = (SeedLoaderRequestSchema as any).parse({ - // ADR-0036 / seed rename: the field is `seeds` (was `datasets`). - seeds: datasets, - config: { - defaultMode: 'upsert', - multiPass: true, - ...(organizationId ? { organizationId } : {}), - }, - }); - const result = await seedLoader.load(request); - seedSummary = { - mode: 'inline', - inserted: result.summary.totalInserted, - updated: result.summary.totalUpdated, - errors: result.errors.length, - // Surface the first write/resolution failure so the - // caller can report WHY nothing landed (e.g. a locked - // DB, a missing table, a failed validation) instead of - // a bare "0 rows". - errorSample: result.errors[0]?.message, - }; - ctx.logger?.info?.(`[MarketplaceInstallLocal] inline seed for ${appId}${organizationId ? ` (org=${organizationId})` : ''}: inserted=${seedSummary.inserted} updated=${seedSummary.updated} errors=${seedSummary.errors}`); + const s = await this.runInlineSeed(ctx, datasets, organizationId); + seedSummary = { mode: 'inline', ...s }; + ctx.logger?.info?.(`[MarketplaceInstallLocal] inline seed for ${appId}${organizationId ? ` (org=${organizationId})` : ''}: inserted=${s.inserted} updated=${s.updated} skipped=${s.skipped} errors=${s.errors}`); } } } catch (err: any) { @@ -845,6 +895,49 @@ export class MarketplaceInstallLocalPlugin implements Plugin { return { translationsLoaded, seeded: seedSummary }; }; + /** + * One SeedLoaderService run over `datasets` (upsert, multi-pass) — the + * shared engine behind install-time seeding, the reseed endpoint and the + * rehydrate-time healer. Throws when objectql/metadata are unavailable; + * per-row write failures are counted into `errors`, not thrown. + */ + private runInlineSeed = async ( + ctx: PluginContext, + datasets: any[], + organizationId?: string, + ): Promise<{ inserted: number; updated: number; skipped: number; errors: number; errorSample?: string }> => { + const ql: any = ctx.getService('objectql'); + let metadata: any; + try { metadata = ctx.getService('metadata'); } catch { /* none */ } + if (!ql || !metadata) throw new Error('objectql-or-metadata-missing'); + + const [{ SeedLoaderService }, { SeedLoaderRequestSchema }] = await Promise.all([ + import('@objectstack/runtime'), + import('@objectstack/spec/data'), + ]); + const seedLoader = new (SeedLoaderService as any)(ql, metadata, ctx.logger); + const request = (SeedLoaderRequestSchema as any).parse({ + // ADR-0036 / seed rename: the field is `seeds` (was `datasets`). + seeds: datasets, + config: { + defaultMode: 'upsert', + multiPass: true, + ...(organizationId ? { organizationId } : {}), + }, + }); + const result = await seedLoader.load(request); + return { + inserted: result.summary.totalInserted, + updated: result.summary.totalUpdated, + skipped: result.summary.totalSkipped ?? 0, + errors: result.errors.length, + // Surface the first write/resolution failure so the caller can + // report WHY nothing landed (e.g. a locked DB, a missing table, + // a failed validation) instead of a bare "0 rows". + errorSample: result.errors[0]?.message, + }; + }; + /** * Best-effort active-org resolution. Reads the better-auth session * (same path as requireAuthenticatedUser) and returns