diff --git a/.changeset/seed-loader-engine-schema-fallback.md b/.changeset/seed-loader-engine-schema-fallback.md new file mode 100644 index 0000000000..6c46fec28f --- /dev/null +++ b/.changeset/seed-loader-engine-schema-fallback.md @@ -0,0 +1,26 @@ +--- +"@objectstack/metadata-protocol": patch +--- + +fix(seed-loader): resolve lookup/master_detail references for objects that only live in the engine registry (marketplace installs) + +`SeedLoaderService.buildDependencyGraph` consulted only `metadata.getObject()` +when building the reference graph. Marketplace-installed packages register +their objects through the `manifest` service straight into the ObjectQL +registry — after the boot-time `bridgeObjectsToMetadataService` pass — so the +metadata service never lists them. The reference graph came back empty for +those objects and every lookup / master_detail seed value was written +verbatim: `crm_contact.crm_account` held the authored natural key +(`"Acme Corporation"`) instead of the target record's id. + +The damage compounded under RLS: `crm_contact` declares +`sharingModel: controlled_by_parent`, whose row filter compiles to a join on +the parent reference. With every reference dangling, the join matched nothing +and the whole object went invisible to everyone — platform admins included — +while the rows sat in the table (REST list `total=0`, single GET 404). + +The loader now falls back to the engine's own schema registry +(feature-detected `engine.getSchema()`, which the ObjectQL engine exposes) +whenever the metadata service has no definition for a seeded object. The +metadata service remains the preferred source; engines without a schema +registry keep the old behavior. diff --git a/packages/cloud-connection/src/marketplace-install-local-seed-lookup.test.ts b/packages/cloud-connection/src/marketplace-install-local-seed-lookup.test.ts new file mode 100644 index 0000000000..c8edce4e78 --- /dev/null +++ b/packages/cloud-connection/src/marketplace-install-local-seed-lookup.test.ts @@ -0,0 +1,242 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Marketplace install MUST resolve seed lookup/master_detail natural keys + * into target record ids. + * + * The bug this pins: marketplace package objects are registered through the + * `manifest` service straight into the ObjectQL registry — AFTER the boot-time + * `bridgeObjectsToMetadataService` pass — so the metadata service never lists + * them. The SeedLoaderService built its reference graph from the metadata + * service only, saw no lookup/master_detail fields for crm_*, and wrote every + * reference value verbatim: `crm_contact.crm_account = 'Acme Corporation'` + * instead of the crm_account row's id. With `sharingModel: + * controlled_by_parent` the dangling parent join then hid EVERY contact from + * EVERY user (REST list total=0, single GET 404) while the rows sat in the DB. + * + * Unlike marketplace-install-local-reseed.test.ts, this test does NOT mock + * @objectstack/runtime — the real SeedLoaderService runs against a faithful + * engine stub whose `getSchema` serves exactly what `manifest.register` + * (ql.registerApp) put into the engine registry, and a metadata service that + * has never heard of the package's objects (the marketplace reality). + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { MarketplaceInstallLocalPlugin } from './marketplace-install-local-plugin.js'; + +type Handler = (c: any) => Promise; + +function makeRawApp() { + const routes = new Map(); + return { + routes, + get: (p: string, h: Handler) => routes.set(`GET ${p}`, h), + post: (p: string, h: Handler) => routes.set(`POST ${p}`, h), + delete: (p: string, h: Handler) => routes.set(`DELETE ${p}`, h), + }; +} + +function makeCtx(rawApp: any, services: Record) { + const hooks = new Map(); + return { + ctx: { + hook: (e: string, h: any) => hooks.set(e, h), + getService: (name: string) => { + if (name === 'http-server') return { getRawApp: () => rawApp }; + const svc = services[name]; + if (svc === undefined) throw new Error(`no ${name}`); + return svc; + }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + }, + fire: async () => { await hooks.get('kernel:ready')?.(); }, + }; +} + +function makeC(body: any, manifestId?: string) { + const json = vi.fn((payload: any, status?: number) => ({ payload, status: status ?? 200 })); + return { + req: { + url: 'http://localhost:3000/api/v1/marketplace/install-local', + raw: new Request('http://localhost:3000/x'), + json: async () => body, + param: (k: string) => (k === 'manifestId' ? manifestId : undefined), + header: () => undefined, + }, + json, + }; +} + +/** Engine stub faithful where it matters: find() filters `where`, insert() + * assigns ids, and `getSchema` reads the registry that `manifest.register` + * fills — the exact surface the real ObjectQL engine gives the seed loader. */ +function makeEngine() { + const store: Record = {}; + const registry: Record = {}; + let idCounter = 0; + const engine: any = { + find: async (objectName: string, query?: any) => { + let records = store[objectName] || []; + if (query?.where) { + records = records.filter((r) => + Object.entries(query.where).every(([k, v]) => r[k] === v), + ); + } + if (typeof query?.limit === 'number') records = records.slice(0, query.limit); + return records; + }, + insert: async (objectName: string, data: any) => { + if (!store[objectName]) store[objectName] = []; + if (Array.isArray(data)) { + const records = data.map((d) => ({ id: `row-${++idCounter}`, ...d })); + store[objectName].push(...records); + return records; + } + const record = { id: `row-${++idCounter}`, ...data }; + store[objectName].push(record); + return record; + }, + update: async (objectName: string, data: any) => { + const records = store[objectName] || []; + const idx = records.findIndex((r) => r.id === data.id); + if (idx >= 0) { + records[idx] = { ...records[idx], ...data }; + return records[idx]; + } + return data; + }, + delete: async () => ({ deleted: 1 }), + count: async (objectName: string) => (store[objectName] || []).length, + aggregate: async () => [], + getSchema: (name: string) => registry[name], + syncSchemas: async () => undefined, + registerApp: (manifest: any) => { + for (const obj of manifest?.objects ?? []) { + if (obj?.name) registry[obj.name] = obj; + } + }, + }; + return { engine, store, registry }; +} + +/** Trimmed from the shipped app.objectstack.hotcrm@2.2.2 artifact: the + * master_detail child under controlled_by_parent whose install corrupted. */ +const HOTCRM_MANIFEST = { + id: 'app.test.hotcrm', + name: 'HotCRM Test', + version: '9.9.9', + objects: [ + { + name: 'crm_account', + label: 'Account', + fields: { + name: { type: 'text', label: 'Name', required: true }, + industry: { type: 'text', label: 'Industry' }, + }, + }, + { + name: 'crm_contact', + label: 'Contact', + sharingModel: 'controlled_by_parent', + fields: { + last_name: { type: 'text', label: 'Last Name', required: true }, + email: { type: 'text', label: 'Email' }, + crm_account: { type: 'master_detail', label: 'Account', reference: 'crm_account', required: true }, + }, + }, + { + name: 'crm_opportunity', + label: 'Opportunity', + fields: { + name: { type: 'text', label: 'Name', required: true }, + crm_account: { type: 'lookup', label: 'Account', reference: 'crm_account', required: true }, + }, + }, + ], + data: [ + { + object: 'crm_account', + externalId: 'name', + mode: 'upsert', + records: [ + { name: 'Acme Corporation', industry: 'technology' }, + { name: 'Globex Inc', industry: 'manufacturing' }, + ], + }, + { + object: 'crm_contact', + externalId: 'email', + mode: 'upsert', + records: [ + { last_name: 'Smith', email: 'john.smith@acme.example.com', crm_account: 'Acme Corporation' }, + { last_name: 'Lee', email: 'maria.lee@globex.example.com', crm_account: 'Globex Inc' }, + ], + }, + { + object: 'crm_opportunity', + externalId: 'name', + mode: 'upsert', + records: [{ name: 'Acme Expansion', crm_account: 'Acme Corporation' }], + }, + ], +}; + +let dir: string; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'mil-seed-lookup-')); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); vi.restoreAllMocks(); }); + +describe('marketplace install — seed lookup resolution', () => { + // Generous timeout: the install handler dynamically imports the real + // @objectstack/runtime (unmocked on purpose), and that cold import alone + // can eat several seconds on a fresh CI runner. + it('writes target record ids (never raw externalId strings) for package objects unknown to the metadata service', { timeout: 30_000 }, async () => { + const { engine, store, registry } = makeEngine(); + const rawApp = makeRawApp(); + const { ctx, fire } = makeCtx(rawApp, { + // The real wiring: manifest.register → ql.registerApp → engine + // registry ONLY. Nothing reaches the metadata service. + manifest: { register: (m: any) => engine.registerApp(m) }, + auth: { api: { getSession: async () => ({ user: { id: 'admin' } }) } }, + objectql: engine, + metadata: { getObject: vi.fn(async () => undefined), list: vi.fn(async () => []) }, + }); + const plugin = new MarketplaceInstallLocalPlugin({ controlPlaneUrl: 'off', storageDir: dir }); + await plugin.start(ctx as any); + await fire(); + + const res = await rawApp.routes.get('POST /api/v1/marketplace/install-local')!( + makeC({ manifest: HOTCRM_MANIFEST }), + ); + expect(res.payload?.success).toBe(true); + // Objects really did register only into the engine registry. + expect(registry.crm_contact).toBeDefined(); + + // The inline seed ran and landed every row. + expect(res.payload?.data?.seeded?.mode).toBe('inline'); + expect(res.payload?.data?.seeded?.errors).toBe(0); + expect(store.crm_account).toHaveLength(2); + expect(store.crm_contact).toHaveLength(2); + expect(store.crm_opportunity).toHaveLength(1); + + // THE regression: reference columns must hold the parent row's id. + const acme = store.crm_account.find((r) => r.name === 'Acme Corporation')!; + const globex = store.crm_account.find((r) => r.name === 'Globex Inc')!; + const john = store.crm_contact.find((r) => r.email === 'john.smith@acme.example.com')!; + const maria = store.crm_contact.find((r) => r.email === 'maria.lee@globex.example.com')!; + const opp = store.crm_opportunity[0]; + + expect(john.crm_account).toBe(acme.id); + expect(maria.crm_account).toBe(globex.id); + expect(opp.crm_account).toBe(acme.id); + + // Belt and braces: no reference column anywhere still carries the + // authored natural-key string. + for (const row of [...store.crm_contact, ...store.crm_opportunity]) { + expect(String(row.crm_account)).not.toMatch(/Acme Corporation|Globex Inc/); + } + }); +}); diff --git a/packages/metadata-protocol/src/seed-loader-engine-schema-fallback.test.ts b/packages/metadata-protocol/src/seed-loader-engine-schema-fallback.test.ts new file mode 100644 index 0000000000..88ef318475 --- /dev/null +++ b/packages/metadata-protocol/src/seed-loader-engine-schema-fallback.test.ts @@ -0,0 +1,235 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect, vi } from 'vitest'; +import { SeedLoaderService } from './seed-loader'; +import type { IDataEngine, IMetadataService } from '@objectstack/spec/contracts'; + +/** + * Reference-graph fallback to the ENGINE schema registry. + * + * Marketplace-installed packages register their objects through the `manifest` + * service straight into the ObjectQL registry (`ql.registerApp`) — AFTER + * `bridgeObjectsToMetadataService` ran at boot — so the metadata service never + * hears about them. `buildDependencyGraph` used to consult ONLY + * `metadata.getObject()`, leaving the reference graph empty for those objects: + * every lookup / master_detail seed value was written verbatim (the raw + * externalId string, e.g. `crm_contact.crm_account = 'Acme Corporation'`) + * instead of the target record's id. Under `controlled_by_parent` RLS a + * dangling parent reference makes the whole child object invisible to + * everyone, platform admins included. + * + * These tests pin the fix: when metadata misses, the loader resolves the + * object definition from the engine's `getSchema()` (feature-detected — the + * ObjectQL engine exposes it; the IDataEngine contract does not require it). + */ + +function createLogger() { + return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; +} + +/** Same faithful engine as seed-loader-replay.test.ts (where-filtering find), + * plus a `getSchema` backed by a mutable schema map — the stand-in for the + * ObjectQL SchemaRegistry that `registerApp` fills. */ +function createFaithfulEngine(schemas: Record) { + const store: Record = {}; + let idCounter = 0; + + const engine = { + find: vi.fn(async (objectName: string, query?: any) => { + let records = store[objectName] || []; + if (query?.where) { + records = records.filter((r) => + Object.entries(query.where).every(([k, v]) => r[k] === v), + ); + } + if (typeof query?.limit === 'number') { + records = records.slice(0, query.limit); + } + return records; + }), + findOne: vi.fn(async (objectName: string, query?: any) => { + const rows = await (engine.find as any)(objectName, { ...query, limit: 1 }); + return rows[0] ?? null; + }), + insert: vi.fn(async (objectName: string, data: any) => { + if (!store[objectName]) store[objectName] = []; + if (Array.isArray(data)) { + const records = data.map((d) => ({ id: `gen-${++idCounter}`, ...d })); + store[objectName].push(...records); + return records; + } + const record = { id: `gen-${++idCounter}`, ...data }; + store[objectName].push(record); + return record; + }), + update: vi.fn(async (objectName: string, data: any) => { + const records = store[objectName] || []; + const idx = records.findIndex((r) => r.id === data.id); + if (idx >= 0) { + records[idx] = { ...records[idx], ...data }; + return records[idx]; + } + return data; + }), + delete: vi.fn(async () => ({ deleted: 1 })), + count: vi.fn(async (objectName: string) => (store[objectName] || []).length), + aggregate: vi.fn(async () => []), + getSchema: vi.fn((objectName: string) => schemas[objectName]), + } as unknown as IDataEngine & { getSchema: ReturnType }; + + return { engine, store }; +} + +/** Mirrors the shipped HotCRM template's shape: a master_detail parent keyed + * by `name`, a child dataset keyed by `email`, plus a self-referencing + * lookup — the exact surfaces the marketplace install corrupted. */ +const ENGINE_SCHEMAS: Record = { + crm_account: { + name: 'crm_account', + fields: { + name: { type: 'text', required: true }, + industry: { type: 'text' }, + }, + }, + crm_contact: { + name: 'crm_contact', + sharingModel: 'controlled_by_parent', + fields: { + last_name: { type: 'text', required: true }, + email: { type: 'text' }, + crm_account: { type: 'master_detail', reference: 'crm_account', required: true }, + reports_to: { type: 'lookup', reference: 'crm_contact' }, + }, + }, + crm_opportunity: { + name: 'crm_opportunity', + fields: { + name: { type: 'text', required: true }, + crm_account: { type: 'lookup', reference: 'crm_account', required: true }, + }, + }, +}; + +/** A metadata service that has never heard of the crm_* objects — the + * marketplace-install reality this regression pins. */ +function createEmptyMetadata(): IMetadataService { + return { + getObject: vi.fn(async () => undefined), + listObjects: vi.fn(async () => []), + register: vi.fn(async () => {}), + get: vi.fn(async () => undefined), + list: vi.fn(async () => []), + unregister: vi.fn(async () => {}), + exists: vi.fn(async () => false), + listNames: vi.fn(async () => []), + } as unknown as IMetadataService; +} + +const CONFIG = { + dryRun: false, + haltOnError: false, + multiPass: true, + defaultMode: 'upsert', + batchSize: 1000, + transaction: false, +} as any; + +const SEEDS = [ + { + object: 'crm_account', + externalId: 'name', + mode: 'upsert', + env: ['prod', 'dev', 'test'], + records: [ + { name: 'Acme Corporation', industry: 'technology' }, + { name: 'Globex Inc', industry: 'manufacturing' }, + ], + }, + { + object: 'crm_contact', + externalId: 'email', + mode: 'upsert', + env: ['prod', 'dev', 'test'], + records: [ + { last_name: 'Smith', email: 'john.smith@acme.example.com', crm_account: 'Acme Corporation' }, + { + last_name: 'Jones', + email: 'sarah.jones@acme.example.com', + crm_account: 'Acme Corporation', + reports_to: 'john.smith@acme.example.com', + }, + ], + }, + { + object: 'crm_opportunity', + externalId: 'name', + mode: 'upsert', + env: ['prod', 'dev', 'test'], + records: [{ name: 'Acme Expansion', crm_account: 'Acme Corporation' }], + }, +] as any[]; + +describe('seed reference graph — engine schema fallback (marketplace objects)', () => { + it('resolves lookup/master_detail natural keys when objects exist only in the engine registry', async () => { + const { engine, store } = createFaithfulEngine(ENGINE_SCHEMAS); + const metadata = createEmptyMetadata(); + + const result = await new SeedLoaderService(engine, metadata, createLogger()).load({ + seeds: SEEDS, + config: CONFIG, + }); + + expect(result.success).toBe(true); + expect(result.summary.totalErrored).toBe(0); + + const acmeId = store.crm_account.find((r) => r.name === 'Acme Corporation')!.id; + const john = store.crm_contact.find((r) => r.email === 'john.smith@acme.example.com')!; + const sarah = store.crm_contact.find((r) => r.email === 'sarah.jones@acme.example.com')!; + const opp = store.crm_opportunity.find((r) => r.name === 'Acme Expansion')!; + + // The corruption this pins against: the raw externalId string landing in + // the reference column. Every reference must be the parent's internal id. + expect(john.crm_account).toBe(acmeId); + expect(sarah.crm_account).toBe(acmeId); + expect(opp.crm_account).toBe(acmeId); + expect(sarah.reports_to).toBe(john.id); + + // Dependency ordering came from the engine schemas too: parent before child. + expect(result.dependencyGraph.insertOrder.indexOf('crm_account')).toBeLessThan( + result.dependencyGraph.insertOrder.indexOf('crm_contact'), + ); + expect(result.summary.totalReferencesResolved).toBeGreaterThanOrEqual(4); + }); + + it('prefers the metadata service definition when it exists (no engine probe)', async () => { + const { engine, store } = createFaithfulEngine({}); + const metadata = createEmptyMetadata(); + (metadata.getObject as any).mockImplementation(async (name: string) => ENGINE_SCHEMAS[name]); + + const result = await new SeedLoaderService(engine, metadata, createLogger()).load({ + seeds: SEEDS, + config: CONFIG, + }); + + expect(result.success).toBe(true); + const acmeId = store.crm_account.find((r) => r.name === 'Acme Corporation')!.id; + expect(store.crm_contact.find((r) => r.email === 'john.smith@acme.example.com')!.crm_account).toBe(acmeId); + expect((engine as any).getSchema).not.toHaveBeenCalled(); + }); + + it('survives an engine without getSchema (contract-minimal engines keep the old behavior)', async () => { + const { engine, store } = createFaithfulEngine({}); + delete (engine as any).getSchema; + const metadata = createEmptyMetadata(); + + const result = await new SeedLoaderService(engine, metadata, createLogger()).load({ + seeds: SEEDS, + config: CONFIG, + }); + + // No schema source at all → no reference graph; values pass through as + // before (this documents, not endorses, the legacy degradation). + expect(result.summary.totalReferencesResolved).toBe(0); + expect(store.crm_contact.every((r) => r.crm_account === 'Acme Corporation')).toBe(true); + }); +}); diff --git a/packages/metadata-protocol/src/seed-loader.ts b/packages/metadata-protocol/src/seed-loader.ts index 3d440e1bd8..ab33b126ad 100644 --- a/packages/metadata-protocol/src/seed-loader.ts +++ b/packages/metadata-protocol/src/seed-loader.ts @@ -154,7 +154,7 @@ export class SeedLoaderService implements ISeedLoaderService { const objectSet = new Set(objectNames); for (const objectName of objectNames) { - const objDef = await this.metadata.getObject(objectName) as any; + const objDef = await this.resolveObjectDefinition(objectName); const dependsOn: string[] = []; const references: ReferenceResolution[] = []; @@ -192,6 +192,33 @@ export class SeedLoaderService implements ISeedLoaderService { return { nodes, insertOrder, circularDependencies }; } + /** + * Object definition for reference-graph construction: the metadata service + * first, then the engine's own schema registry. + * + * The engine fallback matters for marketplace-installed packages: their + * objects are registered through the `manifest` service straight into the + * ObjectQL registry AFTER `bridgeObjectsToMetadataService` ran at boot, so + * the metadata service has never heard of them. Resolving only via metadata + * left the reference graph empty for those objects — every lookup / + * master_detail seed value was written verbatim (the raw externalId string, + * e.g. `crm_contact.crm_account = 'Acme Corporation'`) instead of the target + * record's id. Dangling references break parent joins, and under + * `sharingModel: controlled_by_parent` RLS that makes the whole object + * invisible to everyone, admins included. + */ + private async resolveObjectDefinition(objectName: string): Promise { + const fromMetadata = (await this.metadata.getObject(objectName)) as any; + if (fromMetadata?.fields) return fromMetadata; + try { + const engineSchema = (this.engine as { getSchema?(name: string): unknown }).getSchema?.(objectName) as any; + if (engineSchema?.fields) return engineSchema; + } catch { + // Engine may not expose a schema registry — the metadata result stands. + } + return fromMetadata; + } + async validate(datasets: Seed[], config?: SeedLoaderConfigInput): Promise { const parsedConfig = SeedLoaderConfigSchema.parse({ ...config, dryRun: true }); return this.load({ seeds: datasets, config: parsedConfig });