feat(rest): preserve the original audit timeline for a historical import (#3493)

.changeset/import-historical-audit.md

@@ -0,0 +1,44 @@
+---
+"@objectstack/spec": patch
+"@objectstack/objectql": patch
+"@objectstack/driver-sql": patch
+"@objectstack/rest": patch
+---
+
+feat(rest): `treatAsHistorical` import also preserves the original audit timeline (#3493)
+
+Follow-up to #3479/#3483. `treatAsHistorical` solved the FSM half — mid-lifecycle
+rows are no longer rejected by `initialStates` — but the OTHER half of a historical
+migration, preserving the original timeline, still didn't hold: an imported ticket
+that closed in 2021 stored `updated_at` = the import day (and `updated_by` = the
+importer), and a `writeMode: 'upsert'` refresh silently dropped business `readonly`
+fields (`closed_at`, `resolved_by`). Reports, audit, and "recently modified"
+sorting all came out wrong.
+
+Three layers were force-overwriting the timeline; all three now respect a single
+new opt-in flag, `ExecutionContext.preserveAudit`, which `treatAsHistorical` sets
+alongside `skipStateMachine`:
+
+- **spec**: `ExecutionContext.preserveAudit` (server-set only, never client-supplied)
+  and `DriverOptions.preserveAudit` (threaded to the driver's update stamp).
+- **objectql** — the built-in audit hook (`plugin.ts`) now treats `updated_at` /
+  `updated_by` as CLIENT-PREFERRED (`?? now` / `?? userId`) under `preserveAudit`,
+  symmetric with how `created_at` / `created_by` already behave on insert; and the
+  static-`readonly` write strip (`stripReadonlyFields`) admits a WHITELIST — the
+  audit/timestamp family plus author-declared business `readonly` fields — so an
+  upsert refresh no longer drops them.
+- **driver-sql** — the SQL `update` path keeps a supplied `updated_at` instead of
+  force-advancing it to `now` when `DriverOptions.preserveAudit` is set (fills-only-
+  empty, mirroring the insert stamp).
+- **rest** — the import runner sets `preserveAudit` on the write context iff the
+  request opts into `treatAsHistorical`.
+
+Deliberately a WHITELIST, not the blanket `isSystem` exemption: platform-managed
+`system` columns OUTSIDE the audit family (`organization_id` / tenancy, generated
+columns) STAY stripped, so a historical import reinstates established facts without
+becoming a backdoor to forge tenancy. Permissions / RLS / field-level security are
+unaffected — this changes only which audit/readonly values the runtime overwrites,
+never who may write the record. Fully opt-in: a normal write still auto-stamps
+`updated_at`/`updated_by` and strips `readonly` exactly as before. The objectui
+"Import as historical data" checkbox (objectui#2815) now drives both halves — no new
+UI.

content/docs/references/data/driver.mdx

@@ -96,6 +96,7 @@ const result = DriverCapabilities.parse(data);
 | **traceContext** | `Record<string, string>` | optional | OpenTelemetry context or request ID |
 | **tenantId** | `string` | optional | Tenant Isolation identifier |
 | **timezone** | `string` | optional | Business reference timezone (IANA) for date-dependent generation, e.g. autonumber date tokens |
+| **preserveAudit** | `boolean` | optional | Historical import: keep a supplied updated_at instead of force-stamping now (from ExecutionContext.preserveAudit) |
 
 
 ---

content/docs/references/kernel/execution-context.mdx

@@ -69,6 +69,7 @@ const result = ExecutionContext.parse(data);
 | **skipAutomations** | `boolean` | optional |  |
 | **seedReplay** | `boolean` | optional |  |
 | **skipStateMachine** | `boolean` | optional |  |
+| **preserveAudit** | `boolean` | optional |  |
 | **oauthScopes** | `string[]` | optional |  |
 | **accessToken** | `string` | optional |  |
 | **transaction** | `any` | optional |  |

packages/objectql/src/engine.ts

@@ -782,6 +782,10 @@ export class ObjectQL implements IDataEngine {
       // Propagate the full automation opt-out so `triggerHooks` can skip
       // metadata-bound hooks (import with "run automations" unchecked, undo).
       ...((execCtx as any).skipAutomations ? { skipAutomations: true } : {}),
+      // Propagate the historical-import audit-preservation flag so the built-in
+      // audit hook keeps a client-supplied updated_at/updated_by instead of
+      // stamping now (#3493). Opt-in, server-set only.
+      ...((execCtx as any).preserveAudit ? { preserveAudit: true } : {}),
     } as HookContext['session'];
   }
 
@@ -858,7 +862,8 @@ export class ObjectQL implements IDataEngine {
       !isTenancyDisabled(this._registry.getObject(object));
     const hasTz = execCtx?.timezone !== undefined;
     const isSystem = execCtx?.isSystem === true;
-    if (!hasTx && !hasTenant && !isSystem && !hasTz) return base;
+    const preserveAudit = (execCtx as any)?.preserveAudit === true;
+    if (!hasTx && !hasTenant && !isSystem && !hasTz && !preserveAudit) return base;
     const opts: any = base && typeof base === 'object' ? { ...base } : {};
     if (hasTx && opts.transaction === undefined) {
       opts.transaction = tx;
@@ -877,6 +882,11 @@ export class ObjectQL implements IDataEngine {
       // still flag genuine user-path bugs.
       opts.bypassTenantAudit = true;
     }
+    if (preserveAudit && opts.preserveAudit === undefined) {
+      // Historical import (#3493): let the driver keep a supplied `updated_at`
+      // instead of force-stamping now on the update path.
+      opts.preserveAudit = true;
+    }
     return opts;
   }
 
@@ -2868,7 +2878,7 @@ export class ObjectQL implements IDataEngine {
                // read-only writes are dropped, never the server stamps.
                if (!opCtx.context?.isSystem) {
                    const preRo = hookContext.input.data as Record<string, unknown>;
-                   hookContext.input.data = stripReadonlyFields(updateSchema as any, preRo, suppliedKeys, this.logger) as any;
+                   hookContext.input.data = stripReadonlyFields(updateSchema as any, preRo, suppliedKeys, this.logger, { preserveAudit: opCtx.context?.preserveAudit === true }) as any;
                    reportDroppedFields(preRo, hookContext.input.data as Record<string, unknown>, 'readonly');
                }
                evaluateValidationRules(updateSchema as any, hookContext.input.data as Record<string, unknown>, 'update', { previous: priorRecord, logger: this.logger, currentUser: this.buildEvalUser(opCtx.context), skipStateMachine: shouldSkipStateMachine(opCtx.context) });
@@ -2923,7 +2933,7 @@ export class ObjectQL implements IDataEngine {
                // rejected upstream by the tenant write wall, #2946).
                if (!opCtx.context?.isSystem) {
                    const preRoMulti = hookContext.input.data as Record<string, unknown>;
-                   hookContext.input.data = stripReadonlyFields(updateSchema as any, preRoMulti, suppliedKeys, this.logger) as any;
+                   hookContext.input.data = stripReadonlyFields(updateSchema as any, preRoMulti, suppliedKeys, this.logger, { preserveAudit: opCtx.context?.preserveAudit === true }) as any;
                    reportDroppedFields(preRoMulti, hookContext.input.data as Record<string, unknown>, 'readonly');
                }
                // [#3106] Same enforcement the single-id branch runs at its

packages/objectql/src/plugin.integration.test.ts

@@ -1380,6 +1380,75 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => {
     });
   });
 
+  // #3493 — an opt-in "historical" import (context.preserveAudit) reinstates the
+  // ORIGINAL timeline on UPDATE: the audit hook keeps a client-supplied
+  // updated_at/updated_by (instead of stamping now/importer), and the readonly
+  // strip admits the audit family + author-declared business readonly fields
+  // (closed_at) instead of dropping them. A normal write (no flag) still stamps
+  // and strips. (The driver's own updated_at force-stamp is covered separately in
+  // driver-sql's timestamp-format test — this mock driver echoes the data.)
+  describe('preserveAudit — historical import keeps the original timeline on UPDATE (#3493)', () => {
+    async function bootHistorical() {
+      const updates: Record<string, any>[] = [];
+      const mockDriver = {
+        name: 'hist-capture', version: '1.0.0',
+        connect: async () => {}, disconnect: async () => {},
+        find: async () => [], findOne: async () => null,
+        create: async (_o: string, d: any) => ({ id: 'rec-1', ...d }),
+        update: async (_o: string, _i: any, d: any) => { updates.push({ ...d }); return { id: _i, ...d }; },
+        delete: async () => true, syncSchema: async () => {},
+      };
+      await kernel.use({
+        name: 'hist-capture-plugin', type: 'driver', version: '1.0.0',
+        init: async (ctx) => { ctx.registerService('driver.hist-capture', mockDriver); },
+      });
+      await kernel.use(new ObjectQLPlugin());
+      await kernel.bootstrap();
+      const objectql = kernel.getService('objectql') as any;
+      const obj: ObjectSchema = {
+        name: 'ticket_obj', label: 'Ticket', datasource: 'hist-capture',
+        fields: {
+          name: { name: 'name', label: 'Name', type: 'text' },
+          updated_by: { name: 'updated_by', label: 'Updated By', type: 'text', readonly: true } as any,
+          // author-declared business readonly field — a case's close time
+          closed_at: { name: 'closed_at', label: 'Closed At', type: 'datetime', readonly: true } as any,
+        },
+      };
+      objectql.registry.registerObject(obj, 'test', 'test');
+      return { objectql, updates };
+    }
+
+    it('KEEPS a supplied updated_at / updated_by / closed_at under preserveAudit', async () => {
+      const { objectql, updates } = await bootHistorical();
+      const ts = '2021-03-01T09:00:00.000Z';
+      await objectql.update(
+        'ticket_obj',
+        { id: 'rec-1', name: 'B', updated_at: ts, updated_by: 'u_original', closed_at: ts },
+        { context: { userId: 'u_import', preserveAudit: true } },
+      );
+      expect(updates.length).toBe(1);
+      const data = updates[0];
+      expect(data.name).toBe('B');
+      expect(data.updated_at).toBe(ts);        // hook kept client value (not "now")
+      expect(data.updated_by).toBe('u_original'); // hook kept client value (not importer)
+      expect(data.closed_at).toBe(ts);          // business readonly reinstated, not stripped
+    });
+
+    it('a normal update (no preserveAudit) still overwrites updated_by and strips closed_at', async () => {
+      const { objectql, updates } = await bootHistorical();
+      await objectql.update(
+        'ticket_obj',
+        { id: 'rec-1', name: 'B', closed_at: '2021-03-01T09:00:00.000Z' },
+        { context: { userId: 'u_import' } },
+      );
+      expect(updates.length).toBe(1);
+      const data = updates[0];
+      expect(data.name).toBe('B');
+      expect(data.updated_by).toBe('u_import');   // server-stamped to the actor
+      expect(data).not.toHaveProperty('closed_at'); // readonly business field stripped
+    });
+  });
+
   // #3042 — conditional `readonlyWhen` must be enforced on the BULK
   // (updateMany) path too, not only the single-id path. The bulk strip reads
   // the matched rows' prior state and drops a field locked in ≥1 of them.

packages/objectql/src/plugin.ts

@@ -759,16 +759,22 @@ export class ObjectQLPlugin implements Plugin {
       isInsert: boolean,
     ) => {
       const now = stamp();
+      // A "historical" import (#3493) reinstates the ORIGINAL timeline, so a
+      // client-supplied updated_at/updated_by is CLIENT-PREFERRED here —
+      // symmetric with created_at/created_by on insert — instead of being
+      // overwritten with the import instant. Opt-in and server-set only; a
+      // normal write leaves `preserveAudit` unset and still stamps now.
+      const preserveAudit = session?.preserveAudit === true;
       if (isInsert) {
         record.created_at = record.created_at ?? now;
       }
-      record.updated_at = now;
+      record.updated_at = preserveAudit ? (record.updated_at ?? now) : now;
       if (session?.userId) {
         if (isInsert && hasField(objectName, 'created_by')) {
           record.created_by = record.created_by ?? session.userId;
         }
         if (hasField(objectName, 'updated_by')) {
-          record.updated_by = session.userId;
+          record.updated_by = preserveAudit ? (record.updated_by ?? session.userId) : session.userId;
         }
       }
       // Stamp the driver-layer `tenant_id` column from the caller's active org.

packages/objectql/src/validation/rule-validator.test.ts

@@ -153,6 +153,72 @@ describe('stripReadonlyFields (#2948)', () => {
   });
 });
 
+// #3493 — a "historical" import (preserveAudit) reinstates the original
+// timeline: the audit/timestamp family and author-declared business `readonly`
+// fields survive the strip, while platform-managed `system` columns outside the
+// audit family stay stripped (no tenancy-forging backdoor).
+const historicalFields = {
+  fields: {
+    title: { type: 'text' },
+    created_at: { type: 'datetime', readonly: true, system: true },
+    created_by: { type: 'lookup', readonly: true, system: true },
+    updated_at: { type: 'datetime', readonly: true, system: true },
+    updated_by: { type: 'lookup', readonly: true, system: true },
+    organization_id: { type: 'lookup', readonly: true, system: true },
+    // author-declared business readonly field (NOT system) — e.g. a case's close time
+    closed_at: { type: 'datetime', readonly: true },
+  },
+};
+
+describe('stripReadonlyFields — preserveAudit whitelist (#3493)', () => {
+  it('KEEPS the caller-supplied audit/timestamp family under preserveAudit', () => {
+    const supplied = new Set(['created_at', 'created_by', 'updated_at', 'updated_by']);
+    const data = {
+      created_at: '2020-01-01T00:00:00Z',
+      created_by: 'u_creator',
+      updated_at: '2021-03-01T00:00:00Z',
+      updated_by: 'u_old',
+    };
+    const out = stripReadonlyFields(historicalFields, { ...data }, supplied, undefined, { preserveAudit: true });
+    expect(out).toEqual(data);
+  });
+
+  it('KEEPS an author-declared business readonly field (closed_at) under preserveAudit', () => {
+    const supplied = new Set(['closed_at']);
+    const out = stripReadonlyFields(
+      historicalFields,
+      { closed_at: '2021-03-01T00:00:00Z' },
+      supplied,
+      undefined,
+      { preserveAudit: true },
+    );
+    expect(out).toEqual({ closed_at: '2021-03-01T00:00:00Z' });
+  });
+
+  it('STILL strips a non-audit system column (organization_id) under preserveAudit — no tenancy backdoor', () => {
+    const supplied = new Set(['organization_id', 'closed_at']);
+    const out = stripReadonlyFields(
+      historicalFields,
+      { organization_id: 'org_forged', closed_at: '2021-03-01T00:00:00Z' },
+      supplied,
+      undefined,
+      { preserveAudit: true },
+    );
+    expect(out).toEqual({ closed_at: '2021-03-01T00:00:00Z' });
+  });
+
+  it('strips the whole family as before when preserveAudit is NOT set (regression)', () => {
+    const supplied = new Set(['updated_at', 'closed_at', 'organization_id']);
+    const out = stripReadonlyFields(historicalFields, {
+      title: 'x',
+      updated_at: '2021-03-01T00:00:00Z',
+      closed_at: '2021-03-01T00:00:00Z',
+      organization_id: 'o1',
+    }, supplied);
+    expect(out).toEqual({ title: 'x' });
+  });
+});
+
 describe('needsPriorRecord — field conditional rules (B2)', () => {
   it('is true when a field declares requiredWhen / readonlyWhen', () => {
     expect(needsPriorRecord(invoiceFields as any)).toBe(true);

packages/objectql/src/validation/rule-validator.ts

@@ -325,6 +325,15 @@ export function stripReadonlyWhenFieldsMulti(
  *    system-context writes (import, seed replay, approvals, lifecycle hooks —
  *    all `isSystem: true`) legitimately set read-only columns and skip it.
  *
+ * `options.preserveAudit` (#3493) relaxes the strip for an opt-in "historical"
+ * import that reinstates the original timeline: a caller-supplied read-only
+ * field is KEPT when {@link isPreservableUnderAudit} allows it — the
+ * audit/timestamp family or any author-declared business `readonly` field.
+ * Platform-managed `system` columns outside that family (`organization_id` /
+ * tenancy, generated columns) stay stripped, so the relaxation reinstates facts
+ * without becoming a tenancy-forging backdoor. It is a WHITELIST, deliberately
+ * narrower than the blanket `isSystem` exemption above.
+ *
  * Returns the same object when nothing is stripped, else a shallow copy with the
  * offending keys removed.
  */
@@ -333,21 +342,54 @@ export function stripReadonlyFields(
   data: Record<string, unknown> | undefined | null,
   suppliedKeys: ReadonlySet<string>,
   logger?: EvaluateRulesOptions['logger'],
+  options?: { preserveAudit?: boolean },
 ): Record<string, unknown> | undefined | null {
   const fields = objectSchema?.fields;
   if (!fields || !data) return data;
+  const preserveAudit = options?.preserveAudit === true;
   let result = data;
   for (const [name, def] of Object.entries(fields)) {
     if (!def?.readonly) continue;
     if (!(name in (result as Record<string, unknown>))) continue;
     if (!suppliedKeys.has(name)) continue; // server-stamped, not caller-supplied — keep
+    if (preserveAudit && isPreservableUnderAudit(name, def)) continue; // historical import reinstates it
     if (result === data) result = { ...data };
     delete (result as Record<string, unknown>)[name];
     logger?.warn?.(`Field '${name}' is read-only — ignoring incoming change (#2948)`);
   }
   return result;
 }
 
+/**
+ * The audit / attribution family — the "original timeline" a historical import
+ * (`preserveAudit`) is allowed to reinstate even though these columns are
+ * `system` + `readonly`. Kept in sync with the registry's auto-injected audit
+ * fields (`packages/objectql/src/registry.ts`).
+ */
+const AUDIT_TIMELINE_FIELDS: ReadonlySet<string> = new Set([
+  'created_at',
+  'created_by',
+  'updated_at',
+  'updated_by',
+]);
+
+/**
+ * Whether a caller-supplied `readonly` field may be REINSTATED by an opt-in
+ * historical import (`preserveAudit`), rather than stripped (#3493). Allows:
+ *  - the audit/timestamp family (`created_*` / `updated_*`) — the timeline the
+ *    feature exists to preserve; and
+ *  - author-declared business `readonly` fields (`closed_at`, `resolved_by`, …),
+ *    i.e. anything NOT flagged `system: true`.
+ * Still strips platform-managed `system` columns outside the audit family
+ * (`organization_id` / tenancy, generated columns): a historical import may
+ * reinstate established facts, but must not forge tenancy or system-generated
+ * values.
+ */
+function isPreservableUnderAudit(name: string, def: ConditionalFieldDef): boolean {
+  if (AUDIT_TIMELINE_FIELDS.has(name)) return true;
+  return def.system !== true;
+}
+
 /**
  * A rule needs the prior record if it reasons about the transition or compares
  * against unchanged fields (`state_machine` / `cross_field` / `script`), or if
@@ -383,6 +425,10 @@ interface ConditionalFieldDef {
   readonlyWhen?: string | Expression;
   /** Static, unconditional read-only flag (`field.readonly`). #2948. */
   readonly?: boolean;
+  /** Auto-injected platform/audit field (`field.system`). Distinguishes the
+   *  audit family / tenancy columns from author-declared business fields when
+   *  a historical import (`preserveAudit`) relaxes the readonly strip. #3493. */
+  system?: boolean;
   /** Field type — scopes per-option `visibleWhen` enforcement to choice fields. */
   type?: string;
   /** select/multiselect/radio/checkboxes options; an option may gate itself with `visibleWhen`. */