diff --git a/.changeset/meta-object-read-effective-schema.md b/.changeset/meta-object-read-effective-schema.md new file mode 100644 index 0000000000..8351732725 --- /dev/null +++ b/.changeset/meta-object-read-effective-schema.md @@ -0,0 +1,67 @@ +--- +"@objectstack/metadata-core": minor +"@objectstack/metadata-protocol": minor +"@objectstack/objectql": patch +--- + +fix(metadata-protocol): a `/meta` object read serves the effective runtime schema, whichever layer answered (#6562) + +`GET /api/v1/meta/object/:name` answered a **different set of fields** depending +on which link of its resolution chain produced the answer, for the same object: + +- **registry-backed** → the schema AFTER `applySystemFields`, so it carried the + injected system columns — `created_at`, `created_by`, `updated_at`, + `updated_by`, `organization_id`, `owner_id`, `owning_business_unit_id` — even + when the author declared none of them; +- **overlay-backed** (a `sys_metadata` customization row, or a MetadataService + body) → the stored document VERBATIM, so every one of those columns was simply + absent. + +Whether an object carries an overlay is invisible to the caller, so the same +request reported the platform's own columns or not, and nothing in the response +said which had happened. `/meta` is the machine-readable contract clients and AI +authors code against: an author reading an overlay-backed object saw no +`created_at` / `owner_id` / `organization_id` and reasonably concluded the +columns do not exist — while every one of them is real in the database, +filterable, orderable, and enforced read-only on write. + +**Every `/meta` object read exit now serves the effective schema.** The +single-item read, the list, the cached/ETag branch, both draft reads and the +layered read's `effective` layer all report the injected columns, with the same +`readonly` / `system` markers the engine enforces (`owner_id` stays +`readonly: false` — ownership is transferable). This is the presence half of the +seam #4513 closed the value half of. + +Three things deliberately did **not** change: + +- **`?layers=1`'s `overlay` layer stays byte-verbatim.** Injection happens at the + read exits only, so Studio's "what you customised" diff never shows a column + nobody wrote. Only `effective` is injected. +- **A `GET` → `PUT` round-trip still persists a byte-identical body** (#4326). + The write path gained the strip counterpart: a field byte-identical to the + platform's own definition is removed again on save, so a served document handed + straight back stores exactly what it stored before — same checksum, same + history diff. A declared `owner_id` carrying the author's own label is *not* + the platform's definition and survives untouched. +- **A declared system column stays the author's.** Injection only ever adds a + column nobody declared; it never rewrites one that was. + +Which columns an object carries is `resolveInjectedSystemColumns` +(`@objectstack/spec/data`, #5378) — the same derivation `applySystemFields` +consumes — so every opt-out (`systemFields: false`, `managedBy: 'better-auth'`, +`systemFields.audit`/`.tenant`, `tenancy.enabled: false`, the per-tier +`ownership` table, the `sys_*` namespace) is answered in one place and re-derived +in none. **What** each column looks like moves to `@objectstack/metadata-core` +(`AUDIT_FIELD_DEFS` and the three tenancy/ownership anchors, re-exported from +`@objectstack/objectql` so the symbols still resolve there) — the same relocation, +for the same dependency cycle, as the audit-governance table in #4513: +`@objectstack/objectql` depends on `@objectstack/metadata-protocol`, so the read +path could not import the definitions from the registry that provisions them. +One table now feeds the injection pass and the read exits, so they cannot drift. + +One key is deliberately not carried onto a served document: `organization_id`'s +`indexed`. It is not a `FieldSchema` key — removed in the 16.x line (#2377, +ADR-0049) and rejected by name by the strict schema — and its only consumer is +`driver-mongodb`'s schema builder, which reads the registered schema and never a +served document. It stays at the injection site; that the registry-backed read +answers `_diagnostics: { valid: false }` because of it is filed as #6810. diff --git a/packages/metadata-core/src/index.ts b/packages/metadata-core/src/index.ts index 85882bdfdd..863984b96f 100644 --- a/packages/metadata-core/src/index.ts +++ b/packages/metadata-core/src/index.ts @@ -36,6 +36,15 @@ export * from './engine-update-dispatch.js'; // reporting two. export * from './audit-field-governance.js'; +// [#6562] The injected-system-column DEFINITION table and the served-document +// injection/strip pair built on it, sunk here by the same criterion and for the +// same cycle as the governance table above. `resolveInjectedSystemColumns` +// (spec, #5378) says WHICH columns an object carries; this says WHAT each one +// looks like — the half that used to exist only inside `applySystemFields`, one +// import away from every `/meta` read exit and unreachable from all of them. +// `@objectstack/objectql` now reads this table instead of its own literals. +export * from './injected-system-columns.js'; + // [ADR-0106 / #3682] The metadata-plane FLS projection — one masking function // and one fingerprint, shared by every object-schema exit in // `@objectstack/rest` and `@objectstack/runtime`. Sunk here by the same diff --git a/packages/metadata-core/src/injected-system-columns.ts b/packages/metadata-core/src/injected-system-columns.ts new file mode 100644 index 0000000000..ab4d869a8a --- /dev/null +++ b/packages/metadata-core/src/injected-system-columns.ts @@ -0,0 +1,314 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The **one** table of injected-system-column DEFINITIONS, and the served-document + * injection / strip pair built on it (objectstack#6562, ruled Option B). + * + * ## The split this completes + * + * `resolveInjectedSystemColumns` (`@objectstack/spec/data`, #5378) is the one + * answer to *"WHICH columns does the platform provision on THIS object without + * the author declaring them?"*. It deliberately owns only the names — #3786's + * split leaves *"WHAT does each one look like?"* to the runtime. Until now the + * only copy of that second half lived inside `applySystemFields` + * (`@objectstack/objectql`), reachable only by running the registry. + * + * That is the same wall #4513 hit and recorded one file over + * ({@link applyAuditFieldGovernance}): `@objectstack/objectql` **depends on** + * `@objectstack/metadata-protocol`, so the `/meta` read path cannot import from + * the registry that owns the answer, and the reverse import closes a cycle turbo + * rejects outright. The honest way out is the one this package already carries + * twice — sink the contract into a package **both** sides depend on. This + * package's own dependencies are `{ @objectstack/spec, zod }`, so there is no + * new edge and no new cycle. `applySystemFields` now reads this table instead of + * its own literals; the read path reads it too, and the two cannot drift because + * there is nothing left for them to disagree about. + * + * ## Why a `/meta` read needs it at all (#6562) + * + * `GET /api/v1/meta/object/:name` answered a **different set of fields** + * depending on which link of its resolution chain produced the answer: + * + * - registry-backed → the schema AFTER `applySystemFields`, carrying + * `created_at` / `created_by` / `updated_at` / `updated_by` / + * `organization_id` / `owner_id` / `owning_business_unit_id` even when the + * author declared none of them; + * - overlay-backed (a `sys_metadata` row, or a MetadataService body) → the + * stored document VERBATIM, so every one of those columns was simply absent. + * + * Whether an object carries an overlay is invisible to the caller, so the same + * request reported the platform's own columns or not, and nothing said which had + * happened. An author reading the overlay-backed answer concludes the columns do + * not exist — while every one of them is real in the database, filterable, + * orderable and enforced read-only on write. The maintainer's ruling + * (2026-08-08) is Option B: the read serves the EFFECTIVE runtime schema, and + * the overlay-backed minority path converges on the registry-backed majority. + * + * ## The one key this table deliberately does NOT carry: `indexed` + * + * `applySystemFields` stamps `indexed: ` onto its `organization_id` + * definition, for the MongoDB driver's schema builder (the only consumer; + * `driver-mongodb/src/mongodb-schema.ts`). `indexed` is **not a `FieldSchema` + * key** — it was removed in the 16.x line (#2377, ADR-0049) and `FieldSchema` is + * `strictObject`, so an object document carrying it is rejected BY NAME: + * + * ``` + * Unrecognized key(s) on this field: `indexed`. + * • never a FieldSchema key; a field-level index flag built no index (#2377). + * ``` + * + * Measured on `origin/main` (2026-08-08): a registry-backed `/meta` object read + * therefore already answers `_diagnostics: { valid: false }` on exactly that + * key, in BOTH multiTenant modes — filed as #6810, and deliberately not + * inherited here. Converging the overlay-backed exit onto a key the object + * schema refuses would spread that defect rather than close #6562's; the field + * SET and every spec-authorable key converge, and the DDL hint stays where the + * DDL is. `multiTenant` is also the *only* thing that key depends on, which is + * why nothing in this module takes a `multiTenant` input: per + * `resolveInjectedSystemColumns`' own measurement, the flag changes whether + * `organization_id` is INDEXED, never whether it EXISTS. + */ + +import { + AUDIT_PROVENANCE_FIELDS, + resolveInjectedSystemColumns, + type AuditProvenanceField, +} from '@objectstack/spec/data'; +import { SystemFieldName } from '@objectstack/spec/system'; + +/** + * Column definitions for the audit-provenance family, keyed by the spec's + * {@link AUDIT_PROVENANCE_FIELDS} tuple — the canonical declaration of WHICH + * columns exist (#3786). This table owns only WHAT each column looks like. + * + * The `satisfies` clause is the sync mechanism: a name added to the spec tuple + * without a definition here — or a definition for a name the spec dropped — is + * a compile error, not a silently diverging copy. Same discipline as the spec's + * `APPROVER_VALUE_BINDINGS`. + * + * Moved here from `@objectstack/objectql`'s registry by #6562; see the module + * header for why, and {@link AUDIT_FIELD_GOVERNANCE} for the subset of these + * keys that is forced over a *declared* audit field rather than merely injected + * in its absence. + */ +export const AUDIT_FIELD_DEFS = { + created_at: { + type: 'datetime', + label: 'Created At', + required: false, + readonly: true, + system: true, + description: 'Timestamp when the record was created (auto-populated by the driver).', + }, + created_by: { + type: 'lookup', + reference: 'sys_user', + label: 'Created By', + required: false, + readonly: true, + system: true, + description: 'User who created the record (populated when an authenticated session is present).', + }, + updated_at: { + type: 'datetime', + label: 'Last Modified At', + required: false, + readonly: true, + system: true, + description: 'Timestamp of the most recent modification (auto-populated by the driver).', + }, + updated_by: { + type: 'lookup', + reference: 'sys_user', + label: 'Last Modified By', + required: false, + readonly: true, + system: true, + description: 'User who last modified the record (populated when an authenticated session is present).', + }, +} satisfies Record>; + +/** + * `organization_id` — THE tenant scope anchor, in its **authorable** shape. + * + * ⚠️ `applySystemFields` spreads `indexed: opts.multiTenant` on top of this when + * it provisions the physical column; see the module header for why that key + * lives at the injection site and never in a served document. + */ +export const TENANT_SCOPE_FIELD_DEF: Readonly> = { + type: 'lookup', + reference: 'sys_organization', + label: 'Organization', + required: false, + hidden: true, + readonly: true, + system: true, + description: + 'Tenant scope (auto-populated by org-scoping on insert; NULL on single-tenant stacks).', +}; + +/** + * `owner_id` — the canonical reassignable owner. `system: true` marks it + * platform-provided (so tooling/migrations recognise it), but — unlike the audit + * `*_by` lookups — it is NOT `readonly`: ownership is transferable, so it stays + * editable in forms and assignable via the API. SecurityPlugin auto-stamps it to + * the acting user on insert when left NULL. + */ +export const OWNER_FIELD_DEF: Readonly> = { + type: 'lookup', + reference: 'sys_user', + label: 'Owner', + required: false, + readonly: false, + system: true, + description: + 'Record owner (auto-stamped to the creating user on insert; reassignable). ' + + 'Drives owner-scoped views, reports and notifications.', +}; + +/** + * [ADR-0117 D1] `owning_business_unit_id` — record-level business-unit + * ownership. Shaped after `organization_id` (a server-stamped scope anchor), NOT + * after `owner_id` (a user-assignable business field). The full reasoning for + * each of `readonly` / `hidden` / `required` — and for why the shape presumes + * nothing about the still-unruled D2 policy — stays at the injection site in + * `@objectstack/objectql`'s `applySystemFields`, which is where an author of the + * stamping middleware will be reading. + */ +export const OWNING_BUSINESS_UNIT_FIELD_DEF: Readonly> = { + type: 'lookup', + reference: 'sys_business_unit', + label: 'Owning Business Unit', + required: false, + hidden: true, + readonly: true, + system: true, + description: + 'Record-level business-unit ownership (ADR-0117 D1). Server-stamped scope anchor; ' + + 'NULL until the stamping middleware lands.', +}; + +/** + * The injected columns THIS object carries, as `name -> definition`. + * + * Gated entirely by {@link resolveInjectedSystemColumns} — every opt-out row + * (`systemFields: false`, `managedBy: 'better-auth'`, `systemFields.audit: + * false`, `tenancy.enabled: false`, the per-tier `ownership` table) is answered + * there and re-derived nowhere. `id` is deliberately absent although the plan + * reports it: the primary key is provisioned by the DRIVER + * (`table.string('id').primary()`), not by the injection pass, so no object + * document declares it and neither exit serves it. + * + * Tolerant of bare / un-parsed metadata records, the same contract the plan + * itself carries. + */ +export function injectedSystemColumnDefs(def: unknown): Record>> { + const plan = resolveInjectedSystemColumns(def); + const defs: Record>> = {}; + if (plan.tenant) defs[SystemFieldName.ORGANIZATION_ID] = TENANT_SCOPE_FIELD_DEF; + if (plan.audit) for (const name of AUDIT_PROVENANCE_FIELDS) defs[name] = AUDIT_FIELD_DEFS[name]; + if (plan.owner) defs[SystemFieldName.OWNER_ID] = OWNER_FIELD_DEF; + if (plan.owningBusinessUnit) defs[SystemFieldName.OWNING_BUSINESS_UNIT_ID] = OWNING_BUSINESS_UNIT_FIELD_DEF; + return defs; +} + +/** + * Is this field definition byte-for-byte the platform's own — i.e. a column the + * INJECTION put there, not something the author wrote? + * + * Shallow by construction: every value in the tables above is a primitive, so a + * key-count check plus strict per-key equality is exact. A nested or extra key + * therefore fails the comparison, and failure means "the author's field" — the + * conservative direction, since {@link stripInjectedSystemColumns} only ever + * removes what matches. A declared `owner_id` carrying the author's own label + * survives; one that happens to be identical to the platform definition is + * removed and re-injected identically, which is a no-op by inspection. + */ +function isInjectedDefinition(value: unknown, def: Readonly>): boolean { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const rec = value as Record; + const keys = Object.keys(rec); + if (keys.length !== Object.keys(def).length) return false; + for (const key of keys) if (rec[key] !== def[key]) return false; + return true; +} + +/** The `fields` record of a metadata document, or `undefined` when it has none. */ +function fieldsOf(doc: unknown): Record | undefined { + if (!doc || typeof doc !== 'object' || Array.isArray(doc)) return undefined; + const fields = (doc as Record).fields; + if (!fields || typeof fields !== 'object' || Array.isArray(fields)) return undefined; + return fields as Record; +} + +/** + * Add every injected system column the object carries but does not declare, so a + * served object document reports the EFFECTIVE runtime schema (#6562). + * + * The merge direction is `applySystemFields`': injected definitions **lose** to + * a declared field of the same name, because a declared `owner_id` is the + * author's field and the registry lets it win. (The audit family's *governance* + * — the keys that decide who may write it — is the other half, and stays with + * {@link applyAuditFieldGovernance}: this function only adds absent columns, it + * never rewrites a declared one.) + * + * A document with no `fields` record is returned untouched, deliberately: the + * write-side {@link stripInjectedSystemColumns} could not tell an emptied + * `fields: {}` from one that was never there, and the #4326 byte-identical + * round-trip invariant is what that symmetry protects. + * + * Returns the **same reference** when nothing needed adding, so the + * registry-sourced path (already injected at registration) pays a comparison and + * no copy. Pure and total — any record may be handed to it. + */ +export function applyInjectedSystemColumns(doc: T): T { + const declared = fieldsOf(doc); + if (declared === undefined) return doc; + + let additions: Record | undefined; + for (const [name, def] of Object.entries(injectedSystemColumnDefs(doc))) { + if (declared[name] !== undefined) continue; + additions ??= {}; + additions[name] = { ...def }; + } + if (additions === undefined) return doc; + + return { + ...(doc as unknown as Record), + fields: { ...additions, ...declared }, + } as unknown as T; +} + +/** + * The write-side counterpart of {@link applyInjectedSystemColumns}: remove the + * injected-but-undeclared columns a served document picked up on its way out, so + * the standard Studio GET → edit → PUT round-trip still persists a + * **byte-identical** body (#4326). + * + * Same discipline, and the same reason, as `stripReadDecorations` + * (`@objectstack/spec/kernel`): the write path persists the request body verbatim + * by design (ADR-0005 §Validation), so anything the READ adds must be removed + * again on the way in or it is baked into `sys_metadata.metadata`, into its + * checksum, and into every history diff. It is not the same *list*, though, and + * must not be folded into that one — a read decoration is derived diagnostics + * that no schema accepts, whereas these are real, spec-valid field declarations + * an author may legitimately write. Hence the exactness of + * {@link isInjectedDefinition}: only a field identical to the platform's own is + * removed. + * + * Returns the **same reference** when nothing needed removing. Pure and total. + */ +export function stripInjectedSystemColumns(doc: T): T { + const declared = fieldsOf(doc); + if (declared === undefined) return doc; + + let kept: Record | undefined; + for (const [name, def] of Object.entries(injectedSystemColumnDefs(doc))) { + if (!isInjectedDefinition(declared[name], def)) continue; + kept ??= { ...declared }; + delete kept[name]; + } + if (kept === undefined) return doc; + + return { ...(doc as unknown as Record), fields: kept } as unknown as T; +} diff --git a/packages/metadata-protocol/src/protocol.injected-system-columns.test.ts b/packages/metadata-protocol/src/protocol.injected-system-columns.test.ts new file mode 100644 index 0000000000..b77134db9d --- /dev/null +++ b/packages/metadata-protocol/src/protocol.injected-system-columns.test.ts @@ -0,0 +1,465 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#6562] Every `/meta` object read exit serves the EFFECTIVE runtime schema — + * the injected system columns included — and the write path takes them back off, + * so nothing a read added is ever persisted. + * + * ## The defect + * + * `GET /api/v1/meta/object/:name` resolves through `sys_metadata` overlay → + * MetadataService → SchemaRegistry, and only the last of those three has been + * through `applySystemFields`. So an overlay-backed answer reported the + * platform's own columns — `created_at`, `created_by`, `updated_at`, + * `updated_by`, `organization_id`, `owner_id`, `owning_business_unit_id` — as + * simply ABSENT, while a registry-backed answer for the same object reported + * them present. Whether an object carries an overlay is invisible to the caller. + * An author reading the overlay-backed answer concludes the columns do not + * exist; every one of them is real in the database, filterable, orderable and + * enforced read-only on write. + * + * Maintainer ruling (2026-08-08), Option B: the read serves the effective + * schema, and the overlay-backed minority path converges on the registry-backed + * majority. The ruling's three implementation constraints are the three + * `describe` blocks below, one pin each: + * + * 1. injection at the READ EXITS only — `?layers=1`'s `overlay` layer stays + * byte-verbatim, so Studio's "what you customised" diff never shows a column + * nobody wrote; + * 2. a write-side STRIP counterpart — the #4326 byte-identical GET → PUT + * round-trip keeps holding; + * 3. `resolveInjectedSystemColumns` (spec, #5378) is the derivation — so every + * opt-out row is answered in one place and re-derived in none. + * + * The cross-package pin that the served answer actually EQUALS the registry's — + * asserted against the real `SchemaRegistry` and the real `applySystemFields`, + * in both `multiTenant` modes — is + * `packages/objectql/src/protocol-meta-effective-schema.test.ts`. It cannot live + * here: `@objectstack/objectql` depends on this package, so this file has no way + * to reach the injection pass it is converging on, and a transcribed literal + * would only compare the fix to a copy of itself. + */ + +import { describe, expect, it } from 'vitest'; +// [#5619] The producer's OWN write-verb dispatch decisions, so the fake engine +// below cannot accept a call ObjectQL refuses. From `@objectstack/metadata-core` +// and not `@objectstack/objectql`: objectql DEPENDS ON this package, so that +// import would close a cycle turbo rejects outright. +import { assertEngineDeleteDispatch, assertEngineUpdateDispatch } from '@objectstack/metadata-core'; +import { AUDIT_PROVENANCE_FIELDS } from '@objectstack/spec/data'; +import { ObjectStackProtocolImplementation } from './index.js'; + +interface Row { + id: string; + type: string; + name: string; + organization_id: string | null; + package_id: string | null; + state: string; + metadata: string; + checksum?: string; + version?: number; +} + +function matches(r: Row, where: Record): boolean { + for (const [k, v] of Object.entries(where)) { + if (v === undefined) continue; + if ((r as any)[k] !== v) return false; + } + return true; +} + +function keyOf(w: Record) { + return `${w.type}|${w.name}|${w.organization_id ?? '__env__'}|${w.state ?? 'active'}|${w.package_id ?? '__nopkg__'}`; +} + +function makeStubEngine() { + const rows = new Map(); + let nextId = 0; + const findRow = (w: Record) => { + for (const [k, r] of rows) if (matches(r, w)) return { key: k, row: r }; + return null; + }; + const engine: any = { + async findOne(_t: string, opts: { where: Record }) { + return findRow(opts.where)?.row ?? null; + }, + async find(_t: string, opts: { where: Record }) { + return Array.from(rows.values()).filter((r) => matches(r, opts.where)); + }, + async insert(table: string, data: Record) { + if (table !== 'sys_metadata') return { id: 'side_table' }; + const row = { id: `r_${++nextId}`, ...(data as any) } as Row; + rows.set(keyOf(data), row); + return { id: row.id }; + }, + async update(table: string, data: Record, opts: { where: Record }) { + assertEngineUpdateDispatch(data, opts); + if (table !== 'sys_metadata') return { id: null }; + const found = findRow(opts.where); + if (!found) return { id: null }; + const merged = { ...found.row, ...(data as any) }; + rows.delete(found.key); + rows.set(keyOf(merged), merged); + return { id: found.row.id }; + }, + async delete(_t: string, opts?: Record) { + assertEngineDeleteDispatch(opts); + return { deleted: 0 }; + }, + async transaction(cb: (ctx: any, info: { owned: boolean }) => Promise): Promise { + return cb(undefined, { owned: true }); + }, + async syncObjectSchema() { /* no DDL in this stub */ }, + registry: { + listItems: () => [], + isPackageDisabled: () => false, + getItem: () => undefined, + registerItem: () => {}, + registerObject: () => {}, + getPackage: () => undefined, + }, + }; + return { engine, rows }; +} + +const storedBody = (rows: Map, name: string) => + JSON.parse(Array.from(rows.values()).find((r) => r.name === name)!.metadata); + +/** + * An ordinary user-authored business object declaring NONE of the system + * columns — the population the issue is about. Runtime-created (no artifact), so + * `object`'s `allowRuntimeCreate: true` makes the overlay row genuinely + * writable: this is the AI-authoring path, and the one where the WRITE half of + * the invariant can be exercised end to end. + */ +const authored = (name: string) => ({ + name, + label: 'Invoice', + fields: { amount: { type: 'currency', label: 'Amount' } }, +}); + +/** The columns the platform provisions on such an object, absent any declaration. */ +const INJECTED = [ + 'created_at', 'created_by', 'updated_at', 'updated_by', + 'organization_id', 'owner_id', 'owning_business_unit_id', +] as const; + +async function seed(name: string, item: Record = authored(name), mode?: 'draft') { + const { engine, rows } = makeStubEngine(); + const protocol = new ObjectStackProtocolImplementation(engine); + await protocol.saveMetaItem({ type: 'object', name, item, ...(mode ? { mode } : {}) }); + return { protocol, rows, engine }; +} + +/** One `/meta` read exit: how a client gets an object document out of the protocol. */ +interface Exit { + readonly label: string; + readonly mode?: 'draft'; + read(p: ObjectStackProtocolImplementation, name: string): Promise | undefined>; +} + +const EXITS: readonly Exit[] = [ + { + label: 'GET /meta/object/:name — single item', + async read(p, name) { return (await p.getMetaItem({ type: 'object', name })).item as any; }, + }, + { + label: 'GET /meta/objects/:name — the PLURAL spelling of the same route (#4432)', + async read(p, name) { return (await (p as any).getMetaItem({ type: 'objects', name })).item; }, + }, + { + label: 'GET /meta/objects — list', + async read(p, name) { + const list: any = await p.getMetaItems({ type: 'object' }); + return (list.items as any[]).find((i) => i?.name === name); + }, + }, + { + label: 'GET /meta/object/:name — cached / ETag branch', + async read(p, name) { return (await p.getMetaItemCached({ type: 'object', name }))?.data as any; }, + }, + { + label: 'GET /meta/object/:name?layers=1 — the `effective` layer', + async read(p, name) { + return (await (p as any).getMetaItemLayered({ type: 'object', name }))?.effective; + }, + }, + { + label: 'GET /meta/object/:name?preview=draft — draft overlaid on active', + mode: 'draft', + async read(p, name) { + return (await p.getMetaItem({ type: 'object', name, previewDrafts: true })).item as any; + }, + }, + { + label: 'GET /meta/object/:name?state=draft — the strict draft read', + mode: 'draft', + async read(p, name) { + return (await p.getMetaItem({ type: 'object', name, state: 'draft' })).item as any; + }, + }, +]; + +describe('[#6562] every /meta object read exit serves the effective schema', () => { + for (const exit of EXITS) { + describe(exit.label, () => { + it('reports the injected system columns the author never declared', async () => { + const { protocol } = await seed('crm_invoice', authored('crm_invoice'), exit.mode); + const served = await exit.read(protocol, 'crm_invoice'); + expect(served, 'the exit must actually resolve the item').toBeDefined(); + + expect(Object.keys(served!.fields).sort()).toEqual(['amount', ...INJECTED].sort()); + }); + + it('carries the same governance markers the registry answer carries', async () => { + // Boundary (c) of the ruling — the `engine-audit-anchor-write` + // pin's contract, restated on the read exits. A column reported + // as present but writable would be the #4513 lie with an extra + // step. + const { protocol } = await seed('crm_invoice', authored('crm_invoice'), exit.mode); + const served = await exit.read(protocol, 'crm_invoice'); + + for (const field of AUDIT_PROVENANCE_FIELDS) { + expect(served!.fields[field], field).toMatchObject({ readonly: true, system: true }); + } + expect(served!.fields.organization_id).toMatchObject({ readonly: true, system: true }); + expect(served!.fields.owning_business_unit_id).toMatchObject({ readonly: true, system: true }); + // Ownership is TRANSFERABLE — `owner_id` is the one injected + // column that is `system` but deliberately NOT `readonly`. + expect(served!.fields.owner_id).toMatchObject({ readonly: false, system: true }); + }); + + it('leaves the author’s own fields exactly as declared', async () => { + const { protocol } = await seed('crm_invoice', authored('crm_invoice'), exit.mode); + const served = await exit.read(protocol, 'crm_invoice'); + expect(served!.fields.amount).toEqual({ type: 'currency', label: 'Amount' }); + expect(served!.label).toBe('Invoice'); + }); + }); + } + + it('the served document is one the spec fully accepts', async () => { + // The read now ADDS field declarations, so the product's own `safeParse` + // over exactly the body it just served is the check that they are + // authorable declarations and not an internal shape leaking out. + const { protocol } = await seed('crm_invoice'); + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'crm_invoice' })).item; + expect(served._diagnostics).toEqual({ valid: true }); + }); + + it('a declared system column stays the AUTHOR’s — injection only ever adds', async () => { + // `applySystemFields`' merge direction: `additions` LOSE to a declared + // field of the same name. Forcing the platform definition over the + // author's `owner_id` would be a different (and unruled) change. + const { protocol } = await seed('crm_asset', { + ...authored('crm_asset'), + fields: { + amount: { type: 'currency', label: 'Amount' }, + owner_id: { type: 'lookup', reference: 'sys_user', label: 'Responsible Engineer' }, + }, + }); + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'crm_asset' })).item; + expect(served.fields.owner_id).toEqual({ + type: 'lookup', reference: 'sys_user', label: 'Responsible Engineer', + }); + }); + + it('a non-object metadata type is never touched', async () => { + // Gated on the metadata TYPE, not on "this document happens to have a + // `fields` key" — nothing stops another type from carrying one, and + // injecting columns there would invent a rule the engine does not have. + // Seeded straight into `sys_metadata`: the subject is the READ path, and + // a `page` body shaped like this is (correctly) refused by the save + // path's schema. + const { engine } = makeStubEngine(); + const protocol = new ObjectStackProtocolImplementation(engine); + await engine.insert('sys_metadata', { + type: 'page', name: 'invoice_page', organization_id: null, package_id: null, + state: 'active', + metadata: JSON.stringify({ name: 'invoice_page', fields: { amount: { type: 'currency' } } }), + }); + const served: any = (await protocol.getMetaItem({ type: 'page', name: 'invoice_page' })).item; + expect(Object.keys(served.fields)).toEqual(['amount']); + }); +}); + +/** + * Ruling constraint 3: `resolveInjectedSystemColumns` (#5378) is the derivation. + * Every row of its table is an object the platform provisions NOTHING (or less) + * on, and the read must report exactly that — an injected column reported on an + * object that does not carry one is the same lie pointing the other way. + */ +describe('[#6562] the opt-out rows are the spec derivation’s, re-derived nowhere', () => { + const cases: ReadonlyArray, readonly string[]]> = [ + ['systemFields: false — the hard opt-out (seed/migration tables)', + { systemFields: false }, []], + ['managedBy: better-auth — better-auth owns the column layout', + { managedBy: 'better-auth' }, []], + ['systemFields.audit: false — no audit family', + { systemFields: { audit: false } }, + ['organization_id', 'owner_id', 'owning_business_unit_id']], + ['systemFields.tenant: false — no tenant anchor', + { systemFields: { tenant: false } }, + ['created_at', 'created_by', 'updated_at', 'updated_by', 'owner_id', 'owning_business_unit_id']], + ['tenancy.enabled: false — the schema-level shared-catalog declaration', + { tenancy: { enabled: false } }, + ['created_at', 'created_by', 'updated_at', 'updated_by', 'owner_id', 'owning_business_unit_id']], + ['ownership: org — no per-record ownership anchor, either tier', + { ownership: 'org' }, + ['created_at', 'created_by', 'updated_at', 'updated_by', 'organization_id']], + ['ownership: none — likewise', + { ownership: 'none' }, + ['created_at', 'created_by', 'updated_at', 'updated_by', 'organization_id']], + ]; + + for (const [label, opts, expected] of cases) { + it(label, async () => { + const { engine } = makeStubEngine(); + const protocol = new ObjectStackProtocolImplementation(engine); + // Seeded directly: several of these bodies are deliberately shaped to + // exercise the derivation rather than the save path's schema. + await engine.insert('sys_metadata', { + type: 'object', name: 'crm_invoice', organization_id: null, package_id: null, + state: 'active', + metadata: JSON.stringify({ ...authored('crm_invoice'), ...opts }), + }); + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'crm_invoice' })).item; + expect(Object.keys(served.fields).sort()).toEqual(['amount', ...expected].sort()); + }); + } + + it('the `sys_*` namespace carries no ownership anchor', async () => { + const { engine } = makeStubEngine(); + const protocol = new ObjectStackProtocolImplementation(engine); + await engine.insert('sys_metadata', { + type: 'object', name: 'sys_widget', organization_id: null, package_id: null, + state: 'active', metadata: JSON.stringify(authored('sys_widget')), + }); + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'sys_widget' })).item; + expect(Object.keys(served.fields).sort()).toEqual( + ['amount', 'created_at', 'created_by', 'updated_at', 'updated_by', 'organization_id'].sort(), + ); + }); + + it('the driver-provisioned primary key is NOT injected as a field', async () => { + // `resolveInjectedSystemColumns` reports `id` unconditionally — it is + // addressable on every object — but the injection PASS does not create a + // field for it (`table.string('id').primary()` is the driver's), so + // neither exit may serve one or the two answers diverge again. + const { protocol } = await seed('crm_invoice'); + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'crm_invoice' })).item; + expect(served.fields.id).toBeUndefined(); + }); +}); + +/** + * Ruling constraint 1: injection happens at the read EXITS only. The `overlay` + * layer is Studio's "what you customised" diff — an injected column appearing + * there would report a customization nobody made. + */ +describe('[#6562] ?layers=1 — `overlay` stays byte-verbatim, `effective` is the runtime schema', () => { + it('the overlay layer is exactly the stored row; the effective layer is injected', async () => { + const { protocol } = await seed('crm_invoice'); + const layered: any = await (protocol as any).getMetaItemLayered({ + type: 'object', name: 'crm_invoice', + }); + + expect(Object.keys(layered.overlay.fields)).toEqual(['amount']); + for (const c of INJECTED) { + expect(layered.overlay.fields[c], `${c} must be absent from the overlay layer`).toBeUndefined(); + } + expect(Object.keys(layered.effective.fields).sort()).toEqual(['amount', ...INJECTED].sort()); + }); + + it('the overlay layer is byte-identical to what was persisted', async () => { + const { protocol, rows } = await seed('crm_invoice'); + const layered: any = await (protocol as any).getMetaItemLayered({ + type: 'object', name: 'crm_invoice', + }); + expect(layered.overlay).toEqual(storedBody(rows, 'crm_invoice')); + }); +}); + +/** + * Ruling constraint 2: the write side keeps a strip counterpart, so #4326's + * "a GET → PUT round-trip persists a byte-identical body" keeps holding. + * `protocol.read-decorations.test.ts` owns that invariant for the DECORATIONS; + * these are the overlay-backed object cases it did not have, because before this + * change a read added no field declarations to strip. + */ +describe('[#6562] the write path takes back exactly what the read added (#4326)', () => { + it('GET → PUT the whole served document stores no injected column', async () => { + const { protocol, rows } = await seed('crm_invoice'); + const firstStored = storedBody(rows, 'crm_invoice'); + + // What Studio actually holds: the SERVED document, injected columns and + // decorations included. + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'crm_invoice' })).item; + expect(Object.keys(served.fields).sort()).toEqual(['amount', ...INJECTED].sort()); // precondition + + // Edit one label and PUT the whole thing back, as the designer does. + await protocol.saveMetaItem({ + type: 'object', name: 'crm_invoice', item: { ...served, label: 'Invoice (edited)' }, + }); + + const stored = storedBody(rows, 'crm_invoice'); + expect(Object.keys(stored.fields)).toEqual(['amount']); + expect('_diagnostics' in stored).toBe(false); + expect(stored.label).toBe('Invoice (edited)'); + // Everything except the edited key is byte-identical to the first save. + expect({ ...stored, label: firstStored.label }).toEqual(firstStored); + }); + + it('keeps the checksum stable across an injection-only round-trip', async () => { + const { protocol, rows } = await seed('crm_lead', authored('crm_lead')); + const before = Array.from(rows.values()).find((r) => r.name === 'crm_lead')!.checksum; + + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'crm_lead' })).item; + await protocol.saveMetaItem({ type: 'object', name: 'crm_lead', item: served }); + + expect(Array.from(rows.values()).find((r) => r.name === 'crm_lead')!.checksum).toBe(before); + }); + + it('a draft read PUT back is equally clean', async () => { + const { protocol, rows } = await seed('crm_quote', authored('crm_quote'), 'draft'); + const draft: any = (await protocol.getMetaItem({ + type: 'object', name: 'crm_quote', previewDrafts: true, + })).item; + expect(draft._draft).toBe(true); + expect(draft.fields.created_at).toBeDefined(); + + await protocol.saveMetaItem({ type: 'object', name: 'crm_quote', item: draft, mode: 'draft' }); + + const stored = storedBody(rows, 'crm_quote'); + expect(Object.keys(stored.fields)).toEqual(['amount']); + expect('_draft' in stored).toBe(false); + }); + + it('the strip removes ONLY the platform’s own definition, never the author’s field', async () => { + // The exactness that makes this a strip and not a blocklist: a declared + // `owner_id` is a real, spec-valid authored declaration, and deleting it + // because it shares a name would destroy an author's work on every save. + const item = { + ...authored('crm_asset'), + fields: { + amount: { type: 'currency', label: 'Amount' }, + owner_id: { type: 'lookup', reference: 'sys_user', label: 'Responsible Engineer' }, + }, + }; + const { protocol, rows } = await seed('crm_asset', item); + const served: any = (await protocol.getMetaItem({ type: 'object', name: 'crm_asset' })).item; + await protocol.saveMetaItem({ type: 'object', name: 'crm_asset', item: served }); + + const stored = storedBody(rows, 'crm_asset'); + expect(Object.keys(stored.fields).sort()).toEqual(['amount', 'owner_id']); + expect(stored.fields.owner_id.label).toBe('Responsible Engineer'); + }); + + it('a write that never saw a read is untouched', async () => { + // The strip must be a no-op on an ordinary authored body, or it would be + // rewriting documents nobody served. + const { rows } = await seed('crm_invoice'); + expect(storedBody(rows, 'crm_invoice')).toEqual(authored('crm_invoice')); + }); +}); diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index b8ecf64926..de71ce9ccb 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -16,7 +16,16 @@ import { evaluateRuntimeAuthoringGate } from './runtime-authoring-gate.js'; // ADR-0120 D4 reporting that replaced this file's empty `catch` blocks. import { ensureMetadataOverlayIndexes } from './migrations/overlay-index.js'; import { SysMetadataRepository, type SysMetadataEngine } from './sys-metadata-repository.js'; -import { ConflictError, assertProtocolCompat, applyAuditFieldGovernance, type MetadataItem } from '@objectstack/metadata-core'; +import { + ConflictError, + assertProtocolCompat, + applyAuditFieldGovernance, + // [#6562] The injection/strip pair over the shared injected-column + // definition table — see {@link governServedItem} / {@link stripServedSystemColumns}. + applyInjectedSystemColumns, + stripInjectedSystemColumns, + type MetadataItem, +} from '@objectstack/metadata-core'; // [#5532] One vocabulary of "which driver read errors are benign", shared with // `sys-metadata-repository.ts` in this package and with `DatabaseLoader` in // `@objectstack/metadata` (#5108). See `rethrowUnlessMetadataStoreUnprovisioned`. @@ -140,30 +149,76 @@ function canonicalizeMetaRequestType(request: T): T } /** - * [#4513] The last thing every `/meta` read does to an OBJECT document before - * it leaves this service: make the field metadata it reports agree with what - * the engine enforces on the write path. + * The last thing every `/meta` read does to an OBJECT document before it leaves + * this service: make the field metadata it reports agree with what the engine + * enforces on the write path — in BOTH of the ways it used to disagree. * * The mismatch this closes is structural, not incidental. A `/meta` object read * resolves through `sys_metadata` overlay → MetadataService → SchemaRegistry, - * and only the last of those three has been through `applySystemFields` — so - * the two stored layers answered with whatever the artifact/overlay body - * happened to declare, while `ObjectQL.update` was stripping caller writes to - * the audit family off the registry's post-injection schema. `created_at` read - * `readonly: false` and wrote as read-only, on the same field, at the same - * moment, from the one face a client can actually see (#4447 fixed the write - * half; this is the read half). + * and only the last of those three has been through `applySystemFields`, so the + * answer a caller got depended on which link produced it — with nothing in the + * response saying which one had. Two halves, filed and ruled separately: + * + * - **[#4513] the VALUE half.** The two stored layers answered with whatever + * the artifact/overlay body happened to declare, while `ObjectQL.update` was + * stripping caller writes to the audit family off the registry's + * post-injection schema. `created_at` read `readonly: false` and wrote as + * read-only, on the same field, at the same moment, from the one face a + * client can actually see (#4447 fixed the write half; this is the read + * half). {@link applyAuditFieldGovernance} normalizes a DECLARED audit field. + * - **[#6562] the PRESENCE half.** The stored layers reported the platform's + * own injected columns — `created_at`, `owner_id`, `organization_id`, + * `owning_business_unit_id`, … — as simply ABSENT, so an author reading an + * overlay-backed object reasonably concluded the columns do not exist, while + * every one of them is real in the database, filterable, orderable and + * enforced read-only on write. Maintainer ruling (2026-08-08), Option B: the + * read serves the EFFECTIVE runtime schema and the overlay-backed minority + * converges on the registry-backed majority. + * {@link applyInjectedSystemColumns} adds an UNDECLARED injected column. + * + * The two are composed rather than folded, because they do different things to + * different fields: governance rewrites what the author declared, injection only + * ever adds what nobody declared. Both return their input by reference when + * nothing was needed, so the registry-sourced path (injected AND governed at + * registration) and every non-object type pay a comparison and no copy. * * Applied per EXIT rather than inside `decorateMetadataItem`: decoration is a * diagnostics concern whose output `stripReadDecorations` deliberately removes - * again on write, and governance is neither — it is what the document means. + * again on write, and neither of these is that — they are what the document + * means. The read exits are also the ONLY place injection may happen (ruling + * constraint 1): `getMetaItemLayered` calls this on `effective` and never on + * `overlay`, so Studio's "what you customised" diff keeps showing the row the + * author actually stored. * - * `applyAuditFieldGovernance` returns its input by reference when nothing needed - * forcing, so the registry-sourced path (already governed at registration) and - * every non-object type pay a comparison and no copy. + * ⛔ The write path owes this function a counterpart. See + * {@link stripServedSystemColumns} — without it the standard Studio GET → edit → + * PUT round-trip would persist the injected columns into `sys_metadata`, and the + * #4326 byte-identical invariant would break the day this shipped. */ function governServedItem(type: string, item: T): T { - return canonicalMetaType(type) === 'object' ? applyAuditFieldGovernance(item) : item; + if (canonicalMetaType(type) !== 'object') return item; + return applyInjectedSystemColumns(applyAuditFieldGovernance(item)); +} + +/** + * [#6562] The write-path counterpart of {@link governServedItem}'s injection + * half: take the injected-but-undeclared system columns back off a body on its + * way IN, so a served document handed straight back still persists byte-identical. + * + * Exactly the shape, and exactly the reason, of the `stripReadDecorations` call + * beside it in `saveMetaItem` (#4326) — the write path persists the request body + * verbatim by design (ADR-0005 §Validation), so anything the READ adds must come + * off again on the way in or it is baked into `sys_metadata.metadata`, into its + * checksum, and into every history diff. Kept a SEPARATE strip from that one + * rather than folded into `METADATA_READ_DECORATIONS`, because the two lists are + * different in kind: a read decoration is derived diagnostics no schema accepts, + * whereas an injected column is a real, spec-valid field declaration an author + * may legitimately write — so this strip removes only a field byte-identical to + * the platform's own definition, and a declared `owner_id` carrying the author's + * own label survives untouched. + */ +function stripServedSystemColumns(type: string, item: T): T { + return canonicalMetaType(type) === 'object' ? stripInjectedSystemColumns(item) : item; } /** @@ -8021,6 +8076,18 @@ export class ObjectStackProtocolImplementation implements // Placed first so the destructive-change diff, the schema gate, the // authoring gate and the persisted body all see the same document. request.item = stripReadDecorations(request.item); + // [#6562] …and OUR OWN injected system columns, for the same reason and + // at the same moment. `governServedItem` now serves the EFFECTIVE object + // schema, so the very same Studio round-trip would otherwise persist + // `created_at` / `owner_id` / `organization_id` / … into a body whose + // author declared none of them — turning the platform's own columns into + // a phantom customization in `sys_metadata`, in the checksum, in every + // history diff, and in the layered read's `overlay` layer. Placed + // alongside the decoration strip so the destructive-change diff, the + // schema gate, the authoring gate and the persisted body all still see + // one document. See {@link stripServedSystemColumns} for why this is a + // separate strip from the decoration list and not another entry in it. + request.item = stripServedSystemColumns(request.type, request.item); // Per-item lifecycle (ADR-0005 §"Drafts"). Default is `'publish'` // (legacy semantics — save goes straight live) to keep callers // that predate the draft/publish split working. Studio's diff --git a/packages/objectql/src/protocol-meta-effective-schema.test.ts b/packages/objectql/src/protocol-meta-effective-schema.test.ts new file mode 100644 index 0000000000..19f04e8a3c --- /dev/null +++ b/packages/objectql/src/protocol-meta-effective-schema.test.ts @@ -0,0 +1,301 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#6562] THE core pin: `GET /api/v1/meta/object/:name` reports the SAME field + * set whichever link of its resolution chain produced the answer. + * + * ## The defect, measured on `origin/main` before the fix + * + * The read resolves through `sys_metadata` overlay → MetadataService → + * SchemaRegistry, and only the last of those three has been through + * `applySystemFields`. So for one object, with one authored body: + * + * ``` + * registry-backed fields: [amount, created_at, created_by, organization_id, + * owner_id, owning_business_unit_id, updated_at, updated_by] + * overlay-backed fields: [amount] + * ``` + * + * Whether an object carries an overlay is invisible to the caller, so the same + * request reported the platform's own columns or not and nothing said which had + * happened. An author reading the overlay-backed answer concludes `created_at` + * and `owner_id` do not exist — while every one of them is real in the database, + * filterable, orderable, and enforced read-only on write. Maintainer ruling + * (2026-08-08), Option B: the read serves the EFFECTIVE runtime schema, and the + * overlay-backed minority path converges on the registry-backed majority. + * + * ## Why this file lives in `@objectstack/objectql` + * + * The convergence is a claim about TWO producers agreeing, and only this package + * has both: the real `SchemaRegistry.registerObject` (hence the real + * `applySystemFields`) on one side, and the real protocol read on the other. A + * pin written in `@objectstack/metadata-protocol` would have to transcribe the + * registry's answer into a literal and would then be comparing the fix to a copy + * of itself. Here the registry-backed answer is produced by the engine, live, in + * both `multiTenant` modes — so the day the injection table and the injection + * pass disagree, this fails. + * + * The exit-by-exit and boundary pins (`?layers=1`, the #4326 round-trip, the + * opt-out rows) live next to the read path itself, in + * `packages/metadata-protocol/src/protocol.injected-system-columns.test.ts`. + */ + +import { describe, it, expect } from 'vitest'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +// [#5619] The producer's OWN write-verb dispatch decisions, so the fake engine +// below cannot accept a call ObjectQL refuses. +import { assertEngineDeleteDispatch, assertEngineUpdateDispatch } from '@objectstack/metadata-core'; +import { SchemaRegistry } from './registry.js'; + +interface Row { + id: string; + type: string; + name: string; + organization_id: string | null; + package_id: string | null; + state: string; + metadata: string; + checksum?: string; + version?: number; +} + +function matches(r: Row, where: Record): boolean { + for (const [k, v] of Object.entries(where)) { + if (v === undefined) continue; + if ((r as any)[k] !== v) return false; + } + return true; +} + +function keyOf(w: Record) { + return `${w.type}|${w.name}|${w.organization_id ?? '__env__'}|${w.state ?? 'active'}|${w.package_id ?? '__nopkg__'}`; +} + +/** + * A `sys_metadata` store over a real {@link SchemaRegistry} — the two layers + * whose disagreement is the subject. + */ +function makeHost(multiTenant: boolean) { + const registry = new SchemaRegistry({ multiTenant }); + const rows = new Map(); + let nextId = 0; + const findRow = (w: Record) => { + for (const [k, r] of rows) if (matches(r, w)) return { key: k, row: r }; + return null; + }; + const engine: any = { + registry, + async findOne(_t: string, opts: { where: Record }) { + return findRow(opts.where)?.row ?? null; + }, + async find(_t: string, opts: { where: Record }) { + return Array.from(rows.values()).filter((r) => matches(r, opts.where)); + }, + async insert(table: string, data: Record) { + if (table !== 'sys_metadata') return { id: 'side_table' }; + const row = { id: `r_${++nextId}`, ...(data as any) } as Row; + rows.set(keyOf(data), row); + return { id: row.id }; + }, + async update(table: string, data: Record, opts: { where: Record }) { + assertEngineUpdateDispatch(data, opts); + if (table !== 'sys_metadata') return { id: null }; + const found = findRow(opts.where); + if (!found) return { id: null }; + const merged = { ...found.row, ...(data as any) }; + rows.delete(found.key); + rows.set(keyOf(merged), merged); + return { id: found.row.id }; + }, + async delete(_t: string, opts?: Record) { + assertEngineDeleteDispatch(opts); + return { deleted: 0 }; + }, + async transaction(cb: (ctx: any, info: { owned: boolean }) => Promise): Promise { + return cb(undefined, { owned: true }); + }, + async syncObjectSchema() { /* no DDL in this stub */ }, + async count() { return 0; }, + async aggregate() { return []; }, + }; + return { registry, engine, rows, protocol: new ObjectStackProtocolImplementation(engine) }; +} + +/** + * An ordinary user-authored business object that declares NONE of the system + * columns — the case the issue is about. `ownership` is left omitted so the + * default (`user`) tier applies and both ownership anchors are injected. + */ +const authored = () => ({ + name: 'crm_invoice', + label: 'Invoice', + fields: { amount: { type: 'currency', label: 'Amount' } }, +}); + +const namesOf = (doc: any): string[] => Object.keys(doc.fields).sort(); + +/** + * Every `(field, key)` on which the two answers disagree. + * + * Computed rather than enumerated: the test below asserts the WHOLE list, so a + * NEW divergence — a key one producer grows and the other does not — fails here + * by name instead of hiding behind an assertion that only checked the keys + * somebody thought to list. + */ +function divergences(a: any, b: any): string[] { + const out: string[] = []; + const fields = new Set([...Object.keys(a.fields), ...Object.keys(b.fields)]); + for (const f of [...fields].sort()) { + const fa = (a.fields[f] ?? {}) as Record; + const fb = (b.fields[f] ?? {}) as Record; + for (const k of [...new Set([...Object.keys(fa), ...Object.keys(fb)])].sort()) { + if (fa[k] !== fb[k]) out.push(`${f}.${k}`); + } + } + return out; +} + +describe.each([true, false])('[#6562] /meta object read — effective schema (multiTenant: %s)', (multiTenant) => { + /** + * Read the object registry-backed, then let an overlay row shadow it and read + * again. ONE object, ONE authored body, one endpoint — which is exactly the + * sentence the issue makes, so it is exactly what the fixture reproduces. + */ + async function bothAnswers() { + const host = makeHost(multiTenant); + host.registry.registerObject(authored() as any, 'app.crm'); + + const registryBacked: any = (await host.protocol.getMetaItem({ + type: 'object', name: 'crm_invoice', + })).item; + const registryBackedList: any = ((await host.protocol.getMetaItems({ type: 'object' })).items as any[]) + .find((i) => i?.name === 'crm_invoice'); + + // The customization row: the SAME authored body, stored verbatim, and + // from here on it wins over the registry entry it shadows. + // + // Seeded straight into `sys_metadata` rather than through + // `saveMetaItem` — the same reason #4513's `page` row is, and the reason + // is the point of this file: the subject here is the READ path, and + // `object` carries `allowOrgOverride: false`, so a save over an + // ARTIFACT-backed object is (correctly) refused with `NOT_OVERRIDABLE` + // and going through it would only measure that refusal. The rows this + // shape stands for are real and reachable three ways: a runtime-created + // (artifact-free) object — `allowRuntimeCreate: true`, the AI-authoring + // path the issue's "why it matters" is about — a MetadataService body, + // and any deployment listing `object` in `OS_METADATA_WRITABLE`. The + // WRITE half is exercised on the runtime-created path in + // `packages/metadata-protocol/src/protocol.injected-system-columns.test.ts`. + await host.engine.insert('sys_metadata', { + type: 'object', name: 'crm_invoice', organization_id: null, package_id: null, + state: 'active', metadata: JSON.stringify(authored()), + }); + + const overlayBacked: any = (await host.protocol.getMetaItem({ + type: 'object', name: 'crm_invoice', + })).item; + const overlayBackedList: any = ((await host.protocol.getMetaItems({ type: 'object' })).items as any[]) + .find((i) => i?.name === 'crm_invoice'); + + return { host, registryBacked, registryBackedList, overlayBacked, overlayBackedList }; + } + + it('getMetaItem: the overlay-backed answer reports the registry-backed field set', async () => { + const { registryBacked, overlayBacked } = await bothAnswers(); + + // The precondition, stated so a reader can see the fixture is the real + // case: the author declared exactly one field. + expect(Object.keys(authored().fields)).toEqual(['amount']); + + expect(namesOf(overlayBacked)).toEqual(namesOf(registryBacked)); + expect(namesOf(overlayBacked)).toEqual([ + 'amount', + 'created_at', 'created_by', + 'organization_id', + 'owner_id', 'owning_business_unit_id', + 'updated_at', 'updated_by', + ]); + }); + + it('getMetaItems: the LIST exit converges too', async () => { + const { registryBackedList, overlayBackedList } = await bothAnswers(); + expect(overlayBackedList, 'the list must actually serve the object').toBeDefined(); + expect(namesOf(overlayBackedList)).toEqual(namesOf(registryBackedList)); + }); + + it('the injected columns carry the SAME metadata the registry answer carries', async () => { + const { registryBacked, overlayBacked } = await bothAnswers(); + + // The whole disagreement, computed. `organization_id.indexed` is the ONE + // residual entry and it is not this issue's: `indexed` is not a + // `FieldSchema` key at all — removed in the 16.x line (#2377, ADR-0049) + // and rejected BY NAME by the strict schema — so `applySystemFields` + // stamping it is why a registry-backed read already answers + // `_diagnostics: { valid: false }` (pinned below, filed as #6810). Its + // only consumer is `driver-mongodb`'s schema builder, which reads the + // REGISTERED schema and never a served document. Converging the served + // answer onto a key the object schema refuses would have spread that + // defect instead of closing this one. + expect(divergences(registryBacked, overlayBacked)).toEqual(['organization_id.indexed']); + + // …and the markers the `engine-audit-anchor-write` pin is about, spelled + // out so a failure names the contract rather than a key list. + for (const f of ['created_at', 'created_by', 'updated_at', 'updated_by'] as const) { + expect(overlayBacked.fields[f], f).toMatchObject({ readonly: true, system: true }); + } + expect(overlayBacked.fields.organization_id).toMatchObject({ + type: 'lookup', reference: 'sys_organization', readonly: true, system: true, hidden: true, + }); + // Ownership is TRANSFERABLE, so `owner_id` is `system` but deliberately + // not `readonly` — the one injected column that is not. + expect(overlayBacked.fields.owner_id).toMatchObject({ + type: 'lookup', reference: 'sys_user', readonly: false, system: true, + }); + expect(overlayBacked.fields.owning_business_unit_id).toMatchObject({ + type: 'lookup', reference: 'sys_business_unit', readonly: true, system: true, hidden: true, + }); + }); + + it('multiTenant moves the INDEX, never the field set', async () => { + // `resolveInjectedSystemColumns` says so in its header and + // `injected-system-columns-parity.test.ts` pins it for the injection + // pass; this is the same fact one layer up, on the SERVED document — + // which is why nothing in the read path takes a `multiTenant` input. + const { registryBacked, overlayBacked } = await bothAnswers(); + expect(namesOf(registryBacked)).toContain('organization_id'); + expect(overlayBacked.fields.organization_id.indexed).toBeUndefined(); + expect(registryBacked.fields.organization_id.indexed).toBe(multiTenant); + }); + + it('the served correction never becomes a phantom customization', async () => { + // The read reports the EFFECTIVE schema; `sys_metadata` keeps what the + // author actually stored. Collapsing the two would make the layered + // read's `code` vs `overlay` diff report seven columns nobody declared, + // and would break the #4326 byte-identical round-trip. + const { host } = await bothAnswers(); + await host.protocol.getMetaItem({ type: 'object', name: 'crm_invoice' }); + await host.protocol.getMetaItems({ type: 'object' }); + const stored = JSON.parse(Array.from(host.rows.values()).find((r) => r.name === 'crm_invoice')!.metadata); + expect(Object.keys(stored.fields)).toEqual(['amount']); + // …and the object the registry holds is still the registry's own, not a + // read-hydrated copy of the overlay. + expect(Object.keys((host.registry.getObject('crm_invoice') as any).fields).sort()) + .toContain('organization_id'); + }); + + it('[residual, filed as #6810] only the registry-backed answer fails its own schema', async () => { + // Not a defect this PR introduces and not one it papers over: the + // registry stamps `indexed`, `FieldSchema` rejects it by name, so the + // registry-backed exit has been answering `valid: false` on every + // multi-tenant-capable object since #4001 closed the schema. Pinned in + // both directions so #6810's fix at the injection site has to come back + // and flip these lines, rather than leave a stale expectation behind. + const { registryBacked, overlayBacked } = await bothAnswers(); + expect(overlayBacked._diagnostics).toEqual({ valid: true }); + expect(registryBacked._diagnostics.valid).toBe(false); + expect(registryBacked._diagnostics.errors[0]).toMatchObject({ + path: 'fields.organization_id', + code: 'unrecognized_keys', + }); + }); +}); diff --git a/packages/objectql/src/registry.ts b/packages/objectql/src/registry.ts index a098346621..7c40263a95 100644 --- a/packages/objectql/src/registry.ts +++ b/packages/objectql/src/registry.ts @@ -1,9 +1,16 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. -import { ServiceObject, ObjectSchema, ObjectOwnership, provisionPrimary, resolveCrudAffordances, resolveInjectedSystemColumns, LEGACY_API_METHODS, AUDIT_PROVENANCE_FIELDS, type AuditProvenanceField } from '@objectstack/spec/data'; -// [#4513] The audit-family governance table — see the re-export below for why -// it lives in a package both `objectql` and `metadata-protocol` depend on. -import { AUDIT_FIELD_GOVERNANCE } from '@objectstack/metadata-core'; +import { ServiceObject, ObjectSchema, ObjectOwnership, provisionPrimary, resolveCrudAffordances, resolveInjectedSystemColumns, LEGACY_API_METHODS, AUDIT_PROVENANCE_FIELDS } from '@objectstack/spec/data'; +// [#4513] The audit-family governance table, and [#6562] the injected-column +// DEFINITION tables it governs — see the re-exports below for why both live in a +// package `objectql` and `metadata-protocol` both depend on. +import { + AUDIT_FIELD_GOVERNANCE, + AUDIT_FIELD_DEFS, + TENANT_SCOPE_FIELD_DEF, + OWNER_FIELD_DEF, + OWNING_BUSINESS_UNIT_FIELD_DEF, +} from '@objectstack/metadata-core'; import { SystemFieldName } from '@objectstack/spec/system'; import { resolveTenancyPosture, resolveSearchPinyinEnabled } from '@objectstack/types'; import { postureEnforcesWall } from '@objectstack/spec/security'; @@ -250,51 +257,21 @@ export interface SchemaRegistryOptions { * site for why that shape presumes nothing about the undecided D2 policy. */ /** - * Column definitions for the audit-provenance family, keyed by the spec's - * {@link AUDIT_PROVENANCE_FIELDS} tuple — the canonical declaration of WHICH - * columns exist (#3786). This table owns only WHAT each column looks like. + * [#6562] The column-definition tables — `AUDIT_FIELD_DEFS` and the three + * ownership/tenancy anchors below — now live in `@objectstack/metadata-core`, + * re-exported here so the symbols still resolve from `@objectstack/objectql`. * - * The `satisfies` clause is the sync mechanism: a name added to the spec tuple - * without a definition here — or a definition for a name the spec dropped — is - * a compile error, not a silently diverging copy. Same discipline as the - * spec's `APPROVER_VALUE_BINDINGS`. + * Same criterion, same package and the same cycle as {@link AUDIT_FIELD_GOVERNANCE} + * one comment down: the `/meta` READ path lives in + * `@objectstack/metadata-protocol`, which `@objectstack/objectql` depends on, so + * it could not import WHAT each injected column looks like from the registry + * that provisions it. Until it could, an overlay-backed read served the stored + * document verbatim and reported the platform's own columns as *absent* while a + * registry-backed read of the same object reported them present — one endpoint, + * two field sets, and nothing telling the caller which it had received. The read + * exits and this injection now derive one answer from one table. */ -const AUDIT_FIELD_DEFS = { - created_at: { - type: 'datetime', - label: 'Created At', - required: false, - readonly: true, - system: true, - description: 'Timestamp when the record was created (auto-populated by the driver).', - }, - created_by: { - type: 'lookup', - reference: 'sys_user', - label: 'Created By', - required: false, - readonly: true, - system: true, - description: 'User who created the record (populated when an authenticated session is present).', - }, - updated_at: { - type: 'datetime', - label: 'Last Modified At', - required: false, - readonly: true, - system: true, - description: 'Timestamp of the most recent modification (auto-populated by the driver).', - }, - updated_by: { - type: 'lookup', - reference: 'sys_user', - label: 'Last Modified By', - required: false, - readonly: true, - system: true, - description: 'User who last modified the record (populated when an authenticated session is present).', - }, -} satisfies Record>; +export { AUDIT_FIELD_DEFS, TENANT_SCOPE_FIELD_DEF, OWNER_FIELD_DEF, OWNING_BUSINESS_UNIT_FIELD_DEF }; /** * [#4447] The subset of {@link AUDIT_FIELD_DEFS} that is NOT authorable — the @@ -429,18 +406,15 @@ export function applySystemFields( const overrides: Record = {}; if (wantTenant && !schema.fields?.organization_id) { - additions.organization_id = { - type: 'lookup', - reference: 'sys_organization', - label: 'Organization', - required: false, - indexed: opts.multiTenant, - hidden: true, - readonly: true, - system: true, - description: - 'Tenant scope (auto-populated by org-scoping on insert; NULL on single-tenant stacks).', - }; + // [#6562] The authorable shape is the shared table's; `indexed` is spread on + // top HERE and only here. It is the one key of this definition that is not a + // `FieldSchema` key at all — removed in the 16.x line (#2377, ADR-0049), and + // `FieldSchema` is `strictObject`, so a document carrying it is rejected by + // name ("never a FieldSchema key; a field-level index flag built no index"). + // Its only consumer is `driver-mongodb`'s schema builder, which reads the + // REGISTERED schema and never a served `/meta` document — so it stays at the + // injection site and the served answer converges on everything else. + additions.organization_id = { ...TENANT_SCOPE_FIELD_DEF, indexed: opts.multiTenant }; } if (wantAudit) { @@ -484,17 +458,7 @@ export function applySystemFields( // editable in forms and assignable via the API. SecurityPlugin auto-stamps // it to the acting user on insert when left NULL. if (wantOwner && !schema.fields?.owner_id) { - additions.owner_id = { - type: 'lookup', - reference: 'sys_user', - label: 'Owner', - required: false, - readonly: false, - system: true, - description: - 'Record owner (auto-stamped to the creating user on insert; reassignable). ' + - 'Drives owner-scoped views, reports and notifications.', - }; + additions.owner_id = { ...OWNER_FIELD_DEF }; } // [ADR-0117 D1] Record-level business-unit ownership. Shaped after @@ -529,18 +493,7 @@ export function applySystemFields( // writes and nothing filters is dead weight — the same reasoning that gates // `organization_id`'s index on `multiTenant`. if (wantOwningBusinessUnit && !schema.fields?.[OWNING_BUSINESS_UNIT_FIELD]) { - additions[OWNING_BUSINESS_UNIT_FIELD] = { - type: 'lookup', - reference: 'sys_business_unit', - label: 'Owning Business Unit', - required: false, - hidden: true, - readonly: true, - system: true, - description: - 'Record-level business-unit ownership (ADR-0117 D1). Server-stamped scope anchor; ' + - 'NULL until the stamping middleware lands.', - }; + additions[OWNING_BUSINESS_UNIT_FIELD] = { ...OWNING_BUSINESS_UNIT_FIELD_DEF }; } if (Object.keys(additions).length === 0 && Object.keys(overrides).length === 0) return schema;