From 65940c3b4e9834e491c67dc9b14624c2ede22f7a Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 15:37:38 +1000 Subject: [PATCH 01/20] test(stack): characterise encryptedDynamoDB v2 behaviour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shipping DynamoDB adapter (packages/stack/src/dynamodb, exported as @cipherstash/stack/dynamodb) had zero test coverage. All 861 lines of DynamoDB tests in this repo exercise `protectDynamoDB` in the forked packages/protect-dynamodb, not this one. Pin the current EQL v2 behaviour before the v3 port (#657): - helpers.test.ts (21 tests, pure/offline) — the attribute-mapping contract: the `__source`/`__hmac` split, ste_vec vs scalar envelope reconstruction, nested-field recursion, null/undefined and array handling, deepClone, and error-code fallback. - encrypted-dynamodb.test.ts (22 tests, live) — end-to-end round trips through a real client, bulk operations, audit chaining, FFI error-code propagation, and the __hmac key-condition path: that `encryptQuery` mints exactly the HMAC the item was stored under. No production code changes. This is the baseline the v3 port is verified against. --- .../dynamodb/encrypted-dynamodb.test.ts | 399 ++++++++++++++++++ .../stack/__tests__/dynamodb/helpers.test.ts | 298 +++++++++++++ 2 files changed, 697 insertions(+) create mode 100644 packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts create mode 100644 packages/stack/__tests__/dynamodb/helpers.test.ts diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts new file mode 100644 index 00000000..2cd304fe --- /dev/null +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts @@ -0,0 +1,399 @@ +/** + * Characterisation tests for `encryptedDynamoDB` against a live client. + * + * These pin the CURRENT (EQL v2) end-to-end behaviour of the shipping DynamoDB + * adapter before the EQL v3 port (#657): what `encryptModel` puts on the wire, + * that `decryptModel` reverses it exactly, that the `__hmac` attribute an item + * is stored under is the same value `encryptQuery` mints for a key condition, + * and how failures surface. + * + * There is no DynamoDB in the loop — the adapter never touches the AWS SDK. It + * maps between EQL payloads and DynamoDB attribute names, so these tests assert + * on the attribute map that a caller would hand to `PutCommand`. + * + * Requires CipherStash credentials (`CS_*`), like the rest of this suite. + */ +import 'dotenv/config' +import { beforeAll, describe, expect, it } from 'vitest' +import { encryptedDynamoDB } from '@/dynamodb' +import type { EncryptedDynamoDBInstance } from '@/dynamodb/types' +import type { EncryptionClient } from '@/encryption' +import { Encryption } from '@/index' +import { encryptedColumn, encryptedField, encryptedTable } from '@/schema' + +const users = encryptedTable('users', { + email: encryptedColumn('email').equality(), + name: encryptedColumn('name'), + doc: encryptedColumn('doc').dataType('json').searchableJson('users/doc'), + example: { + protected: encryptedField('example.protected'), + }, +}) + +type User = { + pk: string + email?: string | null + name?: string | null + doc?: Record + role?: string + example?: { protected?: string | null; notProtected?: string } +} + +let client: EncryptionClient +let dynamo: EncryptedDynamoDBInstance + +beforeAll(async () => { + client = await Encryption({ schemas: [users] }) + dynamo = encryptedDynamoDB({ encryptionClient: client }) +}) + +describe('encryptModel', () => { + it('splits equality columns into __source and __hmac, and leaves plaintext alone', async () => { + const result = await dynamo.encryptModel( + { + pk: 'user#1', + email: 'alice@example.com', + name: 'Alice', + role: 'admin', + }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(Object.keys(result.data).sort()).toEqual([ + 'email__hmac', + 'email__source', + 'name__source', + 'pk', + 'role', + ]) + expect(result.data.pk).toBe('user#1') + expect(result.data.role).toBe('admin') + expect(typeof result.data.email__source).toBe('string') + expect(typeof result.data.email__hmac).toBe('string') + // Neither stored attribute leaks the plaintext. + expect(result.data.email__source).not.toContain('alice@example.com') + expect(result.data.email__hmac).not.toContain('alice@example.com') + }) + + it('gives a non-equality column a __source but no __hmac', async () => { + const result = await dynamo.encryptModel( + { pk: 'user#2', name: 'Bob' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(result.data).toHaveProperty('name__source') + expect(result.data).not.toHaveProperty('name__hmac') + }) + + it('stores a searchableJson column as a ste_vec array in __source', async () => { + const result = await dynamo.encryptModel( + { pk: 'user#3', doc: { a: 1, b: 'two' } }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(Array.isArray(result.data.doc__source)).toBe(true) + expect((result.data.doc__source as unknown[]).length).toBeGreaterThan(0) + }) + + it('encrypts a nested field in place, keeping siblings plaintext', async () => { + const result = await dynamo.encryptModel( + { + pk: 'user#4', + example: { protected: 'secret', notProtected: 'public' }, + }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(result.data.example).toEqual({ + protected__source: expect.any(String), + notProtected: 'public', + }) + }) + + it('passes null through without encrypting it', async () => { + const result = await dynamo.encryptModel( + { pk: 'user#5', email: null, name: 'Carol' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(result.data.email).toBeNull() + expect(result.data).not.toHaveProperty('email__source') + }) + + it('does not mutate the caller’s input object', async () => { + const input: User = { pk: 'user#6', email: 'dave@example.com' } + const snapshot = structuredClone(input) + + await dynamo.encryptModel(input, users) + + expect(input).toEqual(snapshot) + }) +}) + +describe('decryptModel', () => { + it('round-trips every column shape back to the original item', async () => { + const original: User = { + pk: 'user#7', + email: 'erin@example.com', + name: 'Erin', + role: 'admin', + example: { protected: 'secret', notProtected: 'public' }, + } + + const encrypted = await dynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await dynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }) + + it('round-trips a searchableJson document', async () => { + const original: User = { pk: 'user#8', doc: { a: 1, b: 'two' } } + + const encrypted = await dynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await dynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }) + + it('tolerates the __hmac attribute being present on the stored item', async () => { + const encrypted = await dynamo.encryptModel( + { pk: 'user#9', email: 'frank@example.com' }, + users, + ) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + expect(encrypted.data).toHaveProperty('email__hmac') + + const decrypted = await dynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data.email).toBe('frank@example.com') + }) +}) + +describe('bulk operations', () => { + it('encrypts and decrypts a batch, preserving order and per-item shape', async () => { + const items: User[] = [ + { pk: 'user#10', email: 'a@example.com', name: 'A' }, + { pk: 'user#11', email: 'b@example.com', name: 'B', role: 'admin' }, + ] + + const encrypted = await dynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + expect(encrypted.data).toHaveLength(2) + for (const item of encrypted.data) { + expect(item).toHaveProperty('email__source') + expect(item).toHaveProperty('email__hmac') + } + + const decrypted = await dynamo.bulkDecryptModels( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(items) + }) + + it('handles heterogeneous items in one batch', async () => { + const items: User[] = [ + { pk: 'user#12', email: 'c@example.com' }, + { pk: 'user#13', name: 'D', example: { protected: 'secret' } }, + ] + + const encrypted = await dynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await dynamo.bulkDecryptModels( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(items) + }) + + it('accepts an empty batch', async () => { + const encrypted = await dynamo.bulkEncryptModels([], users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + expect(encrypted.data).toEqual([]) + }) +}) + +describe('the __hmac key-condition path', () => { + it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { + const email = 'grace@example.com' + + const stored = await dynamo.encryptModel( + { pk: 'user#14', email }, + users, + ) + if (stored.failure) throw new Error(stored.failure.message) + + const term = await client.encryptQuery(email, { + table: users, + column: users.email, + queryType: 'equality', + }) + if (term.failure) throw new Error(term.failure.message) + + // This equality is the whole DynamoDB query story: a caller puts + // `term.data.hm` into `KeyConditionExpression: "email__hmac = :e"` and it + // matches the attribute written at encrypt time. + expect((term.data as { hm: string }).hm).toBe(stored.data.email__hmac) + }) + + it('mints a different HMAC for a different plaintext', async () => { + const term = await client.encryptQuery('someone-else@example.com', { + table: users, + column: users.email, + queryType: 'equality', + }) + if (term.failure) throw new Error(term.failure.message) + + const stored = await dynamo.encryptModel( + { pk: 'user#15', email: 'grace@example.com' }, + users, + ) + if (stored.failure) throw new Error(stored.failure.message) + + expect((term.data as { hm: string }).hm).not.toBe(stored.data.email__hmac) + }) + + it('is deterministic across separate encryptions of the same value', async () => { + const email = 'heidi@example.com' + + const first = await dynamo.encryptModel({ pk: 'a', email }, users) + const second = await dynamo.encryptModel({ pk: 'b', email }, users) + if (first.failure) throw new Error(first.failure.message) + if (second.failure) throw new Error(second.failure.message) + + expect(first.data.email__hmac).toBe(second.data.email__hmac) + // ...while the ciphertext itself is not deterministic. + expect(first.data.email__source).not.toBe(second.data.email__source) + }) +}) + +describe('audit metadata', () => { + it('is accepted on every operation without changing the result', async () => { + const metadata = { sub: 'user-id-123', action: 'characterisation' } + const item: User = { + pk: 'user#16', + email: 'ivan@example.com', + name: 'Ivan', + } + + const encrypted = await dynamo + .encryptModel(item, users) + .audit({ metadata }) + if (encrypted.failure) throw new Error(encrypted.failure.message) + expect(encrypted.data).toHaveProperty('email__hmac') + + const decrypted = await dynamo + .decryptModel(encrypted.data, users) + .audit({ metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data).toEqual(item) + + const bulkEncrypted = await dynamo + .bulkEncryptModels([item], users) + .audit({ metadata }) + if (bulkEncrypted.failure) throw new Error(bulkEncrypted.failure.message) + + const bulkDecrypted = await dynamo + .bulkDecryptModels(bulkEncrypted.data, users) + .audit({ metadata }) + if (bulkDecrypted.failure) throw new Error(bulkDecrypted.failure.message) + expect(bulkDecrypted.data).toEqual([item]) + }) + + it('returns the operation itself so .audit() can be chained before awaiting', () => { + const operation = dynamo.encryptModel({ pk: 'x' }, users) + + expect(operation.audit({ metadata: {} })).toBe(operation) + }) +}) + +describe('error handling', () => { + const unknownColumn = encryptedTable('users', { + nope: encryptedColumn('nonexistent_column'), + }) + + it('surfaces the FFI error code when encrypting an unregistered column', async () => { + const result = await dynamo.encryptModel( + { nope: 'value' }, + // Not among the client's schemas — the FFI rejects the column. + unknownColumn, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) + + it('surfaces the FFI error code on the bulk encrypt path', async () => { + const result = await dynamo.bulkEncryptModels( + [{ nope: 'value' }], + unknownColumn, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) + + it('fails with a DynamoDB error code when __source is not a ciphertext', async () => { + const result = await dynamo.decryptModel( + { email__source: 'not-a-ciphertext' }, + users, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.name).toBe('EncryptedDynamoDBError') + // No FFI code on this path, so the adapter's own fallback code is used. + expect(result.failure?.code).toBe('DYNAMODB_ENCRYPTION_ERROR') + expect(result.failure?.details).toEqual({ context: 'decryptModel' }) + }) + + it('fails on the bulk decrypt path for malformed ciphertext', async () => { + const result = await dynamo.bulkDecryptModels( + [{ email__source: 'not-a-ciphertext' }], + users, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('DYNAMODB_ENCRYPTION_ERROR') + expect(result.failure?.details).toEqual({ context: 'bulkDecryptModels' }) + }) + + it('routes failures to the configured errorHandler', async () => { + const seen: { code: string; message: string }[] = [] + const instrumented = encryptedDynamoDB({ + encryptionClient: client, + options: { + errorHandler: (e) => seen.push({ code: e.code, message: e.message }), + }, + }) + + await instrumented.encryptModel({ nope: 'value' }, unknownColumn) + + expect(seen).toHaveLength(1) + expect(seen[0]?.code).toBe('UNKNOWN_COLUMN') + }) +}) diff --git a/packages/stack/__tests__/dynamodb/helpers.test.ts b/packages/stack/__tests__/dynamodb/helpers.test.ts new file mode 100644 index 00000000..9724941d --- /dev/null +++ b/packages/stack/__tests__/dynamodb/helpers.test.ts @@ -0,0 +1,298 @@ +/** + * Characterisation tests for the pure DynamoDB attribute-mapping helpers. + * + * These pin the CURRENT (EQL v2) behaviour of `toEncryptedDynamoItem` and + * `toItemWithEqlPayloads` before the EQL v3 port (#657). They need no + * credentials and make no network calls — every assertion is about the shape + * of the attribute map on the way into and out of DynamoDB. + * + * Read them as a specification of the storage format: + * `` (an EQL payload) <-> `__source` (+ `__hmac`) + */ +import { describe, expect, it } from 'vitest' +import { + ciphertextAttrSuffix, + deepClone, + EncryptedDynamoDBErrorImpl, + handleError, + searchTermAttrSuffix, + toEncryptedDynamoItem, + toItemWithEqlPayloads, +} from '@/dynamodb/helpers' +import { encryptedColumn, encryptedField, encryptedTable } from '@/schema' + +const users = encryptedTable('users', { + email: encryptedColumn('email').equality(), + name: encryptedColumn('name'), + blob: encryptedColumn('blob').dataType('json'), + doc: encryptedColumn('doc').dataType('json').searchableJson('users/doc'), + example: { + protected: encryptedField('example.protected'), + }, +}) + +const encryptedAttrs = Object.keys(users.build().columns) + +/** A minimal v2 scalar-ciphertext payload as the FFI returns it. */ +const ct = (c: string, hm?: string) => ({ + k: 'ct' as const, + v: 2, + i: { t: 'users', c: 'email' }, + c, + ...(hm ? { hm } : {}), +}) + +/** A minimal v2 ste_vec payload as the FFI returns it. */ +const sv = (entries: unknown[]) => ({ + k: 'sv' as const, + v: 2, + i: { t: 'users', c: 'doc' }, + sv: entries, +}) + +describe('attribute suffixes', () => { + it('are the documented DynamoDB naming convention', () => { + expect(ciphertextAttrSuffix).toBe('__source') + expect(searchTermAttrSuffix).toBe('__hmac') + }) +}) + +describe('toEncryptedDynamoItem (write path)', () => { + it('splits a ciphertext payload with an HMAC into __source and __hmac', () => { + const result = toEncryptedDynamoItem( + { pk: 'user#1', email: ct('ciphertext', 'hmac-value') }, + encryptedAttrs, + ) + + expect(result).toEqual({ + pk: 'user#1', + email__source: 'ciphertext', + email__hmac: 'hmac-value', + }) + // The original attribute name is consumed, not retained alongside. + expect(result).not.toHaveProperty('email') + }) + + it('emits only __source when the payload carries no HMAC', () => { + const result = toEncryptedDynamoItem( + { name: { ...ct('ciphertext'), i: { t: 'users', c: 'name' } } }, + encryptedAttrs, + ) + + expect(result).toEqual({ name__source: 'ciphertext' }) + expect(result).not.toHaveProperty(`name${searchTermAttrSuffix}`) + }) + + it('stores the ste_vec array in __source for a searchable JSON payload', () => { + const entries = [{ s: 'sel', t: 'term' }] + const result = toEncryptedDynamoItem({ doc: sv(entries) }, encryptedAttrs) + + expect(result).toEqual({ doc__source: entries }) + }) + + it('passes attributes absent from the schema through untouched', () => { + const item = { + pk: 'user#1', + role: 'admin', + count: 42, + flag: true, + tags: ['a', 'b'], + } + + expect(toEncryptedDynamoItem(item, encryptedAttrs)).toEqual(item) + }) + + it('preserves null and undefined without adding suffixed attributes', () => { + const result = toEncryptedDynamoItem( + { email: null, name: undefined }, + encryptedAttrs, + ) + + expect(result).toEqual({ email: null, name: undefined }) + }) + + it('recurses into nested objects and splits payloads found inside', () => { + const result = toEncryptedDynamoItem( + { + example: { + protected: { ...ct('nested-ct'), i: { t: 'users', c: 'protected' } }, + notProtected: 'plaintext', + }, + }, + encryptedAttrs, + ) + + expect(result).toEqual({ + example: { + protected__source: 'nested-ct', + notProtected: 'plaintext', + }, + }) + }) + + it('leaves arrays untouched rather than recursing into them', () => { + const result = toEncryptedDynamoItem( + { items: [{ c: 'looks-like-a-payload' }] }, + encryptedAttrs, + ) + + expect(result).toEqual({ items: [{ c: 'looks-like-a-payload' }] }) + }) +}) + +describe('toItemWithEqlPayloads (read path)', () => { + it('rebuilds a v2 scalar ciphertext envelope from __source', () => { + const result = toItemWithEqlPayloads( + { pk: 'user#1', email__source: 'ciphertext' }, + users, + ) + + expect(result).toEqual({ + pk: 'user#1', + email: { + i: { c: 'email', t: 'users' }, + v: 2, + k: 'ct', + c: 'ciphertext', + }, + }) + }) + + it('drops the __hmac attribute — it is not part of the envelope', () => { + const result = toItemWithEqlPayloads( + { email__source: 'ciphertext', email__hmac: 'hmac-value' }, + users, + ) + + expect(Object.keys(result)).toEqual(['email']) + expect(result.email).not.toHaveProperty('hm') + }) + + it('rebuilds a ste_vec envelope for a searchableJson column', () => { + const entries = [{ s: 'sel', t: 'term' }] + const result = toItemWithEqlPayloads({ doc__source: entries }, users) + + expect(result).toEqual({ + doc: { + i: { c: 'doc', t: 'users' }, + v: 2, + k: 'sv', + sv: entries, + }, + }) + }) + + it('rebuilds a scalar envelope for a JSON column without a ste_vec index', () => { + const result = toItemWithEqlPayloads({ blob__source: 'ciphertext' }, users) + + expect(result).toEqual({ + blob: { + i: { c: 'blob', t: 'users' }, + v: 2, + k: 'ct', + c: 'ciphertext', + }, + }) + }) + + it('rebuilds nested payloads, identifying them by the leaf attribute name', () => { + const result = toItemWithEqlPayloads( + { example: { protected__source: 'nested-ct', notProtected: 'plain' } }, + users, + ) + + expect(result).toEqual({ + example: { + protected: { + // NOTE: the identifier uses the leaf name, not the dotted schema key + // (`example.protected`) the column is registered under. + i: { c: 'protected', t: 'users' }, + v: 2, + k: 'ct', + c: 'nested-ct', + }, + notProtected: 'plain', + }, + }) + }) + + it('passes plaintext attributes and null/undefined through untouched', () => { + const item = { pk: 'user#1', role: 'admin', a: null, b: undefined } + + expect(toItemWithEqlPayloads(item, users)).toEqual(item) + }) + + it('round-trips an item through both directions', () => { + const payloads = { + pk: 'user#1', + email: ct('email-ct', 'email-hmac'), + role: 'admin', + } + + const stored = toEncryptedDynamoItem(payloads, encryptedAttrs) + const rebuilt = toItemWithEqlPayloads(stored, users) + + // The HMAC is not recoverable from the rebuilt envelope: it lives only in + // the `__hmac` attribute, which the read path deliberately discards. + expect(rebuilt).toEqual({ + pk: 'user#1', + email: { i: { c: 'email', t: 'users' }, v: 2, k: 'ct', c: 'email-ct' }, + role: 'admin', + }) + }) +}) + +describe('deepClone', () => { + it('returns primitives unchanged', () => { + expect(deepClone(1)).toBe(1) + expect(deepClone('a')).toBe('a') + expect(deepClone(null)).toBe(null) + expect(deepClone(undefined)).toBe(undefined) + }) + + it('clones nested objects and arrays without sharing references', () => { + const source = { a: { b: [1, { c: 2 }] } } + const clone = deepClone(source) + + expect(clone).toEqual(source) + expect(clone.a).not.toBe(source.a) + expect(clone.a.b).not.toBe(source.a.b) + }) +}) + +describe('handleError', () => { + it('falls back to DYNAMODB_ENCRYPTION_ERROR for a plain Error', () => { + const error = handleError(new Error('boom'), 'encryptModel') + + expect(error).toBeInstanceOf(EncryptedDynamoDBErrorImpl) + expect(error.name).toBe('EncryptedDynamoDBError') + expect(error.message).toBe('boom') + expect(error.code).toBe('DYNAMODB_ENCRYPTION_ERROR') + expect(error.details).toEqual({ context: 'encryptModel' }) + }) + + it('preserves a `code` carried on the thrown object', () => { + const thrown = Object.assign(new Error('nope'), { code: 'UNKNOWN_COLUMN' }) + + expect(handleError(thrown, 'encryptModel').code).toBe('UNKNOWN_COLUMN') + }) + + it('stringifies a non-Error throw', () => { + expect(handleError('just a string', 'decryptModel').message).toBe( + 'just a string', + ) + }) + + it('invokes the errorHandler and logger callbacks when supplied', () => { + const seen: unknown[] = [] + const logged: string[] = [] + + const error = handleError(new Error('boom'), 'decryptModel', { + errorHandler: (e) => seen.push(e), + logger: { error: (message) => logged.push(message) }, + }) + + expect(seen).toEqual([error]) + expect(logged).toEqual(['Error in decryptModel']) + }) +}) From d26950d9b925011a9acef65dda0d9e3335123892 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 17:43:45 +1000 Subject: [PATCH 02/20] feat(stack): accept EQL v3 tables in encryptedDynamoDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #657. `encryptedDynamoDB` now accepts tables built with `encryptedTable` + the `types.*` domains from `@cipherstash/stack/v3`, alongside the existing EQL v2 tables. Additive, not a breaking swap: DynamoDB shares none of the v2 Postgres machinery — no EQL extension to install, no migration to run — so there is no reason to force existing callers off v2. The real coupling was on the write path, not where the issue placed it: - `toEncryptedDynamoItem` detected an encrypted value by its `k: 'ct'` tag. EQL v3 scalars carry NO `k` discriminator, so every v3 scalar fell through to the nested-object branch and was written out as a raw map rather than being split into `__source` / `__hmac`. Now keyed off the presence of a ciphertext, with `k: 'sv'` still routing JSON documents. - `toItemWithEqlPayloads` hardcoded `v = 2` and `k: 'ct'`. It now synthesizes the envelope matching the table's version — v3 scalars untagged, JSON documents keeping the mandatory `k: 'sv'`. - Signatures across the four public methods and four operation classes widen to `AnyEncryptedTable`. Not a coupling, contrary to the issue: `cast_as` / `indexes.ste_vec`. A v3 column's `build()` emits the same `{ cast_as, indexes }` shape as a v2 one, so that detection needed no version branch. Client shapes: both are accepted. `EncryptionV3` returns a typed client whose `decryptModel` is a plain Promise taking the table as a second argument; the nominal client returns a chainable operation. The decrypt paths resolve either. Audit metadata on decrypt requires the nominal client — the typed one has no audit surface there, which is documented rather than silently ignored. Capability notes, documented in the skill: - Only equality is usable on DynamoDB. `__hmac` is written for domains that mint an `hm` term (the `*Eq` family plus TextOrd/TextOrdOre/TextSearch). Ordering and bloom-filter terms have no key-condition surface and are not stored, so those columns stay decryptable but not queryable. - EQL v3 has no `encryptedField` equivalent, so nested object encryption still requires a v2 table. Adds 31 tests for the v3 paths (74 total for the adapter, from zero before this branch). The 43 v2 characterisation tests pass unchanged, which is what establishes this as backward compatible. Also corrects two stale claims in AGENTS.md: the package attribution for `encryptedDynamoDB`, and the note that DynamoDB requires EQL v2. --- .changeset/dynamodb-eql-v3.md | 37 ++ AGENTS.md | 4 +- .../dynamodb/encrypted-dynamodb-v3.test.ts | 363 ++++++++++++++++++ .../__tests__/dynamodb/helpers-v3.test.ts | 207 ++++++++++ packages/stack/src/dynamodb/helpers.ts | 114 +++++- packages/stack/src/dynamodb/index.ts | 39 +- .../operations/bulk-decrypt-models.ts | 42 +- .../operations/bulk-encrypt-models.ts | 35 +- .../src/dynamodb/operations/decrypt-model.ts | 42 +- .../src/dynamodb/operations/encrypt-model.ts | 35 +- packages/stack/src/dynamodb/types.ts | 93 ++++- skills/stash-dynamodb/SKILL.md | 167 +++++--- 12 files changed, 1022 insertions(+), 156 deletions(-) create mode 100644 .changeset/dynamodb-eql-v3.md create mode 100644 packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts create mode 100644 packages/stack/__tests__/dynamodb/helpers-v3.test.ts diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md new file mode 100644 index 00000000..bc6335ec --- /dev/null +++ b/.changeset/dynamodb-eql-v3.md @@ -0,0 +1,37 @@ +--- +'@cipherstash/stack': minor +'stash': patch +--- + +`encryptedDynamoDB` now accepts EQL v3 tables. + +Pass a table built with `encryptedTable` + the `types.*` domains from +`@cipherstash/stack/v3` (or `@cipherstash/stack/eql/v3`) to any of +`encryptModel`, `bulkEncryptModels`, `decryptModel`, `bulkDecryptModels`. Both +the typed client from `EncryptionV3` and the nominal client from +`Encryption({ config: { eqlVersion: 3 } })` are accepted. + +EQL v2 tables continue to work unchanged — this is additive, and no existing +caller needs to change. The table decides which wire format is used, so a +DynamoDB table populated under one version must keep being read with that +version. + +This fixes a latent bug that made v3 unusable: the write path detected an +encrypted value by its `k: 'ct'` tag, but EQL v3 scalars carry no `k` +discriminator at all. Every v3 scalar fell through to the nested-object branch +and was written as a raw map instead of being split into `__source` and +`__hmac`. + +Notes on capability: + +- Only equality is usable on DynamoDB. `__hmac` is written for domains + that mint an `hm` term — the `*Eq` family, plus `TextOrd`/`TextOrdOre`/ + `TextSearch`. Ordering and bloom-filter terms have no DynamoDB query surface + and are not stored, so those columns remain decryptable but not queryable. +- EQL v3 has no `encryptedField` equivalent, so nested object encryption + requires an EQL v2 table. +- Audit metadata on `decryptModel` / `bulkDecryptModels` requires the nominal + client; the `EncryptionV3` client has no audit surface on decrypt. + +The DynamoDB adapter also gains its first test coverage — 74 tests across the +v2 and v3 paths, where it previously had none. diff --git a/AGENTS.md b/AGENTS.md index 62b8299c..85ba6444 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,7 +86,7 @@ If these variables are missing, tests that require live encryption will fail or - `packages/stack-supabase`: Supabase integration (`@cipherstash/stack-supabase`), depends on `@cipherstash/stack` — `encryptedSupabase` (v2) and `encryptedSupabaseV3` (v3). Split out of `@cipherstash/stack`. - `packages/schema`: Schema builder utilities and types (`encryptedTable`, `encryptedColumn`, `encryptedField`) - `packages/nextjs`: Next.js helpers and Clerk integration (`./clerk` export) -- `packages/protect-dynamodb`: DynamoDB helpers (`encryptedDynamoDB`) +- `packages/protect-dynamodb`: DynamoDB helpers (`protectDynamoDB`), built on `@cipherstash/protect`. A fork of the shipping adapter, kept for existing consumers of the standalone package; EQL v2 only. The maintained implementation is `packages/stack/src/dynamodb` (`encryptedDynamoDB`) - `packages/utils`: Shared config (`utils/config`) and logger (`utils/logger`) - `packages/bench`: Performance / index-engagement benchmarks (private, not published) - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) @@ -147,7 +147,7 @@ Three rules to remember when editing CI or pnpm config: ## Key Concepts and APIs - **Initialization**: `EncryptionV3({ schemas })` returns the typed EQL v3 client. (`Encryption({ schemas })` is the legacy v2 client.) Provide at least one `encryptedTable`. -- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments and is what DynamoDB (`encryptedField`) still requires; see #657.) +- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments. DynamoDB accepts both v3 and v2 tables; only v2 supports nested fields via `encryptedField`.) - **Operations** (all return Result-like objects and support chaining `.withLockContext(lockContext)` and `.audit()` when applicable): - `encrypt(plaintext, { table, column })` - `decrypt(encryptedPayload)` diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts new file mode 100644 index 00000000..2cbfb8d7 --- /dev/null +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -0,0 +1,363 @@ +/** + * Live end-to-end tests for `encryptedDynamoDB` against EQL v3 tables (#657). + * + * Mirrors the v2 suite in `encrypted-dynamodb.test.ts` and adds the two things + * only v3 raises: + * + * - both client shapes must work. `EncryptionV3` returns a `TypedEncryptionClient` + * whose `decryptModel` is a plain `Promise>` taking the table as a + * second argument; `Encryption({ config: { eqlVersion: 3 } })` returns the + * nominal chainable client. Audit metadata on decrypt only has somewhere to + * go on the latter. + * - per-domain storage: only equality domains mint the `hm` term that backs a + * DynamoDB key condition. Ordering and free-text terms are not stored. + * + * Requires CipherStash credentials (`CS_*`), like the rest of this suite. + */ +import 'dotenv/config' +import { beforeAll, describe, expect, it } from 'vitest' +import { encryptedDynamoDB } from '@/dynamodb' +import type { EncryptedDynamoDBInstance } from '@/dynamodb/types' +import type { EncryptionClient } from '@/encryption' +import { EncryptionV3 } from '@/encryption/v3' +import { encryptedTable, types } from '@/eql/v3' +import { Encryption } from '@/index' + +const users = encryptedTable('users_v3_dynamo', { + email: types.TextEq('email'), + name: types.Text('name'), + age: types.IntegerOrd('age'), + bio: types.TextSearch('bio'), + meta: types.Json('meta'), +}) + +type User = { + pk: string + email?: string + name?: string + age?: number + bio?: string + meta?: Record + role?: string +} + +/** The typed client from `EncryptionV3` — the documented v3 entry point. */ +let typedDynamo: EncryptedDynamoDBInstance +/** The nominal chainable client, forced into v3 mode. */ +let nominalClient: EncryptionClient +let nominalDynamo: EncryptedDynamoDBInstance + +beforeAll(async () => { + const typedClient = await EncryptionV3({ schemas: [users] }) + typedDynamo = encryptedDynamoDB({ encryptionClient: typedClient }) + + nominalClient = await Encryption({ + schemas: [users] as never, + config: { eqlVersion: 3 }, + }) + nominalDynamo = encryptedDynamoDB({ encryptionClient: nominalClient }) +}) + +describe('encryptModel with a v3 table', () => { + it('splits an equality domain into __source and __hmac', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#1', email: 'alice@example.com', role: 'admin' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(Object.keys(result.data).sort()).toEqual([ + 'email__hmac', + 'email__source', + 'pk', + 'role', + ]) + expect(typeof result.data.email__source).toBe('string') + expect(typeof result.data.email__hmac).toBe('string') + expect(result.data.email__source).not.toContain('alice@example.com') + }) + + it('regression: a v3 scalar is split, not written out as a raw map', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#2', name: 'Bob' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + // Before the port, gating on `k === 'ct'` meant untagged v3 scalars fell + // through to the nested-object branch and were stored as `{ name: { v, i, + // c } }` — no __source, and the ciphertext nested one level down. + expect(result.data).toHaveProperty('name__source') + expect(result.data).not.toHaveProperty('name') + expect(typeof result.data.name__source).toBe('string') + }) + + it('gives an ordering domain a __source but no __hmac', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#3', age: 42 }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + // `IntegerOrd` is equality-capable in Postgres via its ordering term, but + // that term has no DynamoDB query surface, so nothing is stored for it. + expect(result.data).toHaveProperty('age__source') + expect(result.data).not.toHaveProperty('age__hmac') + }) + + it('keeps only the equality term of a TextSearch domain', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#4', bio: 'a long biography' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + // TextSearch mints hm + op + bf; only hm is storable as an attribute. + expect(Object.keys(result.data).sort()).toEqual([ + 'bio__hmac', + 'bio__source', + 'pk', + ]) + }) + + it('stores a Json domain as a ste_vec array', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#5', meta: { a: 1, b: { c: 'deep' } } }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(Array.isArray(result.data.meta__source)).toBe(true) + expect((result.data.meta__source as unknown[]).length).toBeGreaterThan(0) + }) +}) + +describe('round trips with a v3 table', () => { + it('round-trips every domain back to the original item', async () => { + const original: User = { + pk: 'user#6', + email: 'erin@example.com', + name: 'Erin', + age: 42, + bio: 'a long biography', + meta: { a: 1, b: { c: 'deep' } }, + role: 'admin', + } + + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await typedDynamo.decryptModel( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }) + + it('round-trips the same item through the nominal client', async () => { + const original: User = { + pk: 'user#7', + email: 'frank@example.com', + age: 7, + meta: { z: 1 }, + } + + const encrypted = await nominalDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await nominalDynamo.decryptModel( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }) + + it('produces items either client can decrypt — the wire format is the same', async () => { + const original: User = { pk: 'user#8', email: 'grace@example.com' } + + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await nominalDynamo.decryptModel( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }) +}) + +describe('bulk operations with a v3 table', () => { + it('encrypts and decrypts a batch', async () => { + const items: User[] = [ + { pk: 'user#9', email: 'a@example.com', name: 'A' }, + { pk: 'user#10', email: 'b@example.com', age: 3 }, + ] + + const encrypted = await typedDynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + expect(encrypted.data).toHaveLength(2) + for (const item of encrypted.data) { + expect(item).toHaveProperty('email__source') + expect(item).toHaveProperty('email__hmac') + } + + const decrypted = await typedDynamo.bulkDecryptModels( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(items) + }) + + it('round-trips a batch through the nominal client', async () => { + const items: User[] = [ + { pk: 'user#11', email: 'c@example.com' }, + { pk: 'user#12', name: 'D' }, + ] + + const encrypted = await nominalDynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await nominalDynamo.bulkDecryptModels( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(items) + }) +}) + +describe('the __hmac key-condition path with a v3 table', () => { + it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { + const email = 'heidi@example.com' + + const stored = await typedDynamo.encryptModel( + { pk: 'user#13', email }, + users, + ) + if (stored.failure) throw new Error(stored.failure.message) + + const term = await nominalClient.encryptQuery(email, { + table: users as never, + column: users.email as never, + }) + if (term.failure) throw new Error(term.failure.message) + + // `encryptQuery` on a v3 equality domain mints the bare term — `{v, i, hm}` + // with no ciphertext — so the key condition consumes `hm` directly. + expect(term.data).not.toHaveProperty('c') + expect((term.data as { hm: string }).hm).toBe(stored.data.email__hmac) + }) + + it('is deterministic across separate encryptions of the same value', async () => { + const email = 'ivan@example.com' + + const first = await typedDynamo.encryptModel( + { pk: 'a', email }, + users, + ) + const second = await typedDynamo.encryptModel( + { pk: 'b', email }, + users, + ) + if (first.failure) throw new Error(first.failure.message) + if (second.failure) throw new Error(second.failure.message) + + expect(first.data.email__hmac).toBe(second.data.email__hmac) + expect(first.data.email__source).not.toBe(second.data.email__source) + }) +}) + +describe('audit metadata with a v3 table', () => { + const metadata = { sub: 'user-id-123', action: 'v3-port' } + + it('is carried on every operation of the nominal client', async () => { + const item: User = { pk: 'user#14', email: 'judy@example.com' } + + const encrypted = await nominalDynamo + .encryptModel(item, users) + .audit({ metadata }) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await nominalDynamo + .decryptModel(encrypted.data, users) + .audit({ metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(item) + }) + + it('is accepted on the typed client, though decrypt cannot carry it', async () => { + const item: User = { pk: 'user#15', email: 'ken@example.com' } + + const encrypted = await typedDynamo + .encryptModel(item, users) + .audit({ metadata }) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + // The typed client's `decryptModel` returns a plain promise with no audit + // surface. The chain must still resolve correctly — the metadata is simply + // not forwarded. Use the nominal client if decrypt audit matters. + const decrypted = await typedDynamo + .decryptModel(encrypted.data, users) + .audit({ metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(item) + }) +}) + +describe('error handling with a v3 table', () => { + const unknown = encryptedTable('users_v3_dynamo', { + nope: types.TextEq('nonexistent_column'), + }) + + it('surfaces the FFI error code for an unregistered column', async () => { + const result = await typedDynamo.encryptModel({ nope: 'value' }, unknown) + + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) + + it('surfaces the FFI ciphertext error code for a malformed __source', async () => { + const result = await typedDynamo.decryptModel( + { email__source: 'not-a-ciphertext' }, + users, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.name).toBe('EncryptedDynamoDBError') + // Behaviour difference from v2, and an improvement: in v2 mode this path + // produced a bare "Unexpected end of input" with no code, so the adapter + // fell back to DYNAMODB_ENCRYPTION_ERROR (see the v2 suite). v3 rejects it + // as "Invalid EQL ciphertext" with a real FFI code, which is propagated. + expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') + expect(result.failure?.details).toEqual({ context: 'decryptModel' }) + }) + + it('routes v3 failures to the configured errorHandler', async () => { + const seen: string[] = [] + const instrumented = encryptedDynamoDB({ + encryptionClient: nominalClient, + options: { errorHandler: (e) => seen.push(e.code) }, + }) + + await instrumented.encryptModel({ nope: 'value' }, unknown) + + expect(seen).toEqual(['UNKNOWN_COLUMN']) + }) +}) diff --git a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts new file mode 100644 index 00000000..81f5fb57 --- /dev/null +++ b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts @@ -0,0 +1,207 @@ +/** + * Pure tests for the EQL v3 attribute mapping (#657). + * + * The v2 equivalents live in `helpers.test.ts` and are the characterisation + * baseline; this file asserts only what v3 changes: + * + * - v3 scalars carry NO `k` discriminator, so the write path must key off the + * presence of a ciphertext. Gating on `k === 'ct'` (as the v2 code did) + * dropped every v3 scalar through to the nested-object branch and wrote it + * out as a raw map. + * - the read path synthesizes a v3 envelope (`v: 3`, no `k`) for a v3 table + * and a v2 one (`v: 2`, `k: 'ct'`) for a v2 table. + * - a v3 JSON document keeps `k: 'sv'`, which is mandatory — deserialization + * fails with "missing field `k`" without it. + */ +import { describe, expect, it } from 'vitest' +import { + isV3Table, + toEncryptedDynamoItem, + toItemWithEqlPayloads, +} from '@/dynamodb/helpers' +import { encryptedTable, types } from '@/eql/v3' +import { + encryptedColumn, + encryptedField, + encryptedTable as encryptedTableV2, +} from '@/schema' + +const users = encryptedTable('users', { + email: types.TextEq('email'), + name: types.Text('name'), + age: types.IntegerOrd('age'), + meta: types.Json('meta'), +}) + +const encryptedAttrs = Object.keys(users.build().columns) + +/** A v3 scalar payload as the FFI returns it — note the absent `k`. */ +const scalar = ( + column: string, + c: string, + terms: Record = {}, +) => ({ + v: 3, + i: { t: 'users', c: column }, + c, + ...terms, +}) + +describe('isV3Table', () => { + it('recognises a v3 table by its concrete-domain columns', () => { + expect(isV3Table(users)).toBe(true) + }) + + it('recognises a flat v2 table', () => { + const v2 = encryptedTableV2('users', { + email: encryptedColumn('email').equality(), + }) + + expect(isV3Table(v2)).toBe(false) + }) + + it('recognises a v2 table whose columns are nested under a group', () => { + const v2 = encryptedTableV2('users', { + example: { protected: encryptedField('example.protected') }, + }) + + expect(isV3Table(v2)).toBe(false) + }) +}) + +describe('toEncryptedDynamoItem with v3 payloads', () => { + it('splits an untagged v3 scalar into __source and __hmac', () => { + const result = toEncryptedDynamoItem( + { + pk: 'user#1', + email: scalar('email', 'ciphertext', { hm: 'hmac-value' }), + }, + encryptedAttrs, + ) + + expect(result).toEqual({ + pk: 'user#1', + email__source: 'ciphertext', + email__hmac: 'hmac-value', + }) + }) + + it('stores a storage-only column with no search term', () => { + const result = toEncryptedDynamoItem( + { name: scalar('name', 'ciphertext') }, + encryptedAttrs, + ) + + expect(result).toEqual({ name__source: 'ciphertext' }) + }) + + it('drops the ordering term of an *Ord domain — DynamoDB cannot use it', () => { + const result = toEncryptedDynamoItem( + { age: scalar('age', 'ciphertext', { op: 'ordering-term' }) }, + encryptedAttrs, + ) + + // Decryptable, but not orderable and not usable in a key condition. + expect(result).toEqual({ age__source: 'ciphertext' }) + expect(result).not.toHaveProperty('age__hmac') + }) + + it('keeps the equality term of a TextSearch domain and drops the rest', () => { + const result = toEncryptedDynamoItem( + { + email: scalar('email', 'ciphertext', { + hm: 'hmac-value', + op: 'ordering-term', + bf: [1, 2, 3], + }), + }, + encryptedAttrs, + ) + + expect(result).toEqual({ + email__source: 'ciphertext', + email__hmac: 'hmac-value', + }) + }) + + it('stores a v3 ste_vec document as its sv array', () => { + const entries = [{ s: 'sel', c: 'ct', a: false, hm: 'h' }] + const result = toEncryptedDynamoItem( + { meta: { v: 3, k: 'sv', i: { t: 'users', c: 'meta' }, sv: entries } }, + encryptedAttrs, + ) + + expect(result).toEqual({ meta__source: entries }) + }) +}) + +describe('toItemWithEqlPayloads for a v3 table', () => { + it('rebuilds an untagged v3 scalar envelope', () => { + const result = toItemWithEqlPayloads({ email__source: 'ciphertext' }, users) + + expect(result).toEqual({ + email: { i: { c: 'email', t: 'users' }, v: 3, c: 'ciphertext' }, + }) + expect(result.email).not.toHaveProperty('k') + }) + + it('rebuilds a v3 ste_vec envelope, keeping the mandatory k tag', () => { + const entries = [{ s: 'sel', c: 'ct' }] + const result = toItemWithEqlPayloads({ meta__source: entries }, users) + + expect(result).toEqual({ + meta: { i: { c: 'meta', t: 'users' }, v: 3, k: 'sv', sv: entries }, + }) + }) + + it('still emits a v2 envelope for a v2 table', () => { + const v2 = encryptedTableV2('users', { + email: encryptedColumn('email').equality(), + }) + + expect(toItemWithEqlPayloads({ email__source: 'ct' }, v2)).toEqual({ + email: { i: { c: 'email', t: 'users' }, v: 2, k: 'ct', c: 'ct' }, + }) + }) + + it('discards __hmac and passes plaintext through, as in v2', () => { + const result = toItemWithEqlPayloads( + { pk: 'user#1', email__source: 'ct', email__hmac: 'h', role: 'admin' }, + users, + ) + + expect(result).toEqual({ + pk: 'user#1', + email: { i: { c: 'email', t: 'users' }, v: 3, c: 'ct' }, + role: 'admin', + }) + }) + + it('always emits both `v` and `i` — a payload missing either is returned raw', () => { + const rebuilt = toItemWithEqlPayloads({ email__source: 'ct' }, users) + .email as Record + + // Not a cosmetic assertion: the FFI treats `v`/`i` as the "is this + // encrypted" detection keys. A payload missing either is silently passed + // through undecrypted rather than erroring. + expect(rebuilt).toHaveProperty('v') + expect(rebuilt).toHaveProperty('i') + }) + + it('round-trips a v3 item through both directions', () => { + const stored = toEncryptedDynamoItem( + { + pk: 'user#1', + email: scalar('email', 'email-ct', { hm: 'email-hmac' }), + role: 'admin', + }, + encryptedAttrs, + ) + + expect(toItemWithEqlPayloads(stored, users)).toEqual({ + pk: 'user#1', + email: { i: { c: 'email', t: 'users' }, v: 3, c: 'email-ct' }, + role: 'admin', + }) + }) +}) diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index 812692d7..c2252fa3 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -1,13 +1,37 @@ import type { ProtectErrorCode } from '@cipherstash/protect-ffi' import { ProtectError as FfiProtectError } from '@cipherstash/protect-ffi' -import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import type { EncryptedValue } from '@/types' import { logger } from '@/utils/logger' -import type { EncryptedDynamoDBError } from './types' +import type { AnyEncryptedTable, EncryptedDynamoDBError } from './types' export const ciphertextAttrSuffix = '__source' export const searchTermAttrSuffix = '__hmac' +/** + * Which EQL wire version to synthesize when rebuilding an envelope from the + * stored `__source` attribute. + * + * v2 columns are `EncryptedColumn` instances from `@/schema`; v3 columns are + * the concrete-domain classes from `@/eql/v3`, which are the only ones carrying + * `getQueryCapabilities`. That method is the discriminator used elsewhere in + * the codebase (see `encryption/helpers/infer-index-type.ts`), so reuse it + * rather than inventing a second signal. + * + * A v2 table's builders may nest (`encryptedField` groups columns under an + * object), so walk one level rather than assuming a flat map. v3 has no nested + * columns at all. + */ +export function isV3Table(table: AnyEncryptedTable): boolean { + return Object.values(table.columnBuilders).some(isV3Column) +} + +function isV3Column(builder: unknown): boolean { + if (builder === null || typeof builder !== 'object') return false + if ('getQueryCapabilities' in builder) return true + // A v2 nested group: `{ ssn: encryptedField(...), address: { ... } }`. + return Object.values(builder as Record).some(isV3Column) +} + export class EncryptedDynamoDBErrorImpl extends Error implements EncryptedDynamoDBError @@ -69,6 +93,51 @@ export function handleError( return dynamoError } +/** + * Resolve a decrypt call against either client shape. + * + * The nominal `EncryptionClient` returns a chainable operation that carries + * `.audit()`; the `TypedEncryptionClient` from `EncryptionV3` returns a plain + * `Promise>`. Chain the audit metadata when the client can carry it, + * otherwise await the promise directly — a typed client has no audit surface + * on decrypt, so the metadata has nowhere to go. + */ +export async function resolveDecryptResult( + operation: unknown, + auditData: { metadata?: Record }, +): Promise< + { data: T; failure?: never } | { data?: never; failure: DecryptFailure } +> { + const chainable = operation as { + audit?: (data: { metadata?: Record }) => unknown + } + + const resolved = + typeof chainable?.audit === 'function' + ? await chainable.audit(auditData) + : await operation + + return resolved as + | { data: T; failure?: never } + | { data?: never; failure: DecryptFailure } +} + +type DecryptFailure = { message: string; code?: string } + +/** + * Rethrow a Result failure as an `Error` that preserves the FFI error code. + * `withResult`'s `ensureError` wraps non-Error objects, which would otherwise + * lose the code before `handleError` can read it. + */ +export function throwPreservingCode(failure: { + message: string + code?: string +}): never { + const error = new Error(failure.message) as Error & { code?: string } + error.code = failure.code + throw error +} + export function deepClone(obj: T): T { if (obj === null || typeof obj !== 'object') { return obj @@ -109,18 +178,31 @@ export function toEncryptedDynamoItem( 'c' in (attrValue as object)) ) { const encryptPayload = attrValue as EncryptedValue - if (encryptPayload?.k === 'ct' && encryptPayload.c) { + + // A JSON document, in either wire version, keeps its `k: 'sv'` tag. Its + // index terms live *inside* the `sv` entries, so the whole array is + // stored and there is no separate search-term attribute to split out. + if (encryptPayload?.k === 'sv' && encryptPayload.sv) { const result: Record = {} - if (encryptPayload.hm) { - result[`${attrName}${searchTermAttrSuffix}`] = encryptPayload.hm - } - result[`${attrName}${ciphertextAttrSuffix}`] = encryptPayload.c + result[`${attrName}${ciphertextAttrSuffix}`] = encryptPayload.sv return result } - if (encryptPayload?.k === 'sv' && encryptPayload.sv) { + // Scalars. v2 tags every payload `k: 'ct'`; v3 scalars carry NO `k` + // discriminator at all, so the presence of a ciphertext is the signal — + // gating on `k === 'ct'` would drop every v3 scalar through to the + // nested-object branch below and write it out as a raw map. + if (encryptPayload?.c) { const result: Record = {} - result[`${attrName}${ciphertextAttrSuffix}`] = encryptPayload.sv + // `hm` is the deterministic equality term, and the only one a DynamoDB + // key condition can use. Ordering terms (`op`/`ob`) and the match + // bloom filter (`bf`) have no DynamoDB query surface, so they are not + // stored — the value stays decryptable, it is just not orderable or + // text-searchable inside DynamoDB. + if (encryptPayload.hm) { + result[`${attrName}${searchTermAttrSuffix}`] = encryptPayload.hm + } + result[`${attrName}${ciphertextAttrSuffix}`] = encryptPayload.c return result } } @@ -154,8 +236,10 @@ export function toEncryptedDynamoItem( export function toItemWithEqlPayloads( decrypted: Record, - encryptionSchema: EncryptedTable, + encryptionSchema: AnyEncryptedTable, ): Record { + const v = isV3Table(encryptionSchema) ? 3 : 2 + function processValue( attrName: string, attrValue: unknown, @@ -180,10 +264,11 @@ export function toItemWithEqlPayloads( (encryptedAttrs.includes(columnName) || isNested) ) { const i = { c: columnName, t: encryptConfig.tableName } - const v = 2 // Nested values are not searchable, so we can just return the standard EQL payload. // Worth noting, that encryptConfig.columns[columnName] will be undefined if isNested is true. + // A v3 column builds the same `{ cast_as, indexes }` shape as a v2 one, + // so this detection needs no version branch. if ( !isNested && encryptConfig.columns[columnName].cast_as === 'json' && @@ -193,17 +278,22 @@ export function toItemWithEqlPayloads( [columnName]: { i, v, + // Mandatory in both versions — a v3 ste_vec document without it + // fails deserialization with "missing field `k`". k: 'sv', sv: attrValue, }, } } + // v3 scalars are untagged; only v2 carries `k: 'ct'`. Both versions + // require `v` and `i` — a payload missing either is not recognized as + // encrypted and is returned to the caller verbatim rather than erroring. return { [columnName]: { i, v, - k: 'ct', + ...(v === 2 ? { k: 'ct' } : {}), c: attrValue, }, } diff --git a/packages/stack/src/dynamodb/index.ts b/packages/stack/src/dynamodb/index.ts index 4d54ad53..1f11764f 100644 --- a/packages/stack/src/dynamodb/index.ts +++ b/packages/stack/src/dynamodb/index.ts @@ -1,10 +1,10 @@ -import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import type { EncryptedValue } from '@/types' import { BulkDecryptModelsOperation } from './operations/bulk-decrypt-models' import { BulkEncryptModelsOperation } from './operations/bulk-encrypt-models' import { DecryptModelOperation } from './operations/decrypt-model' import { EncryptModelOperation } from './operations/encrypt-model' import type { + AnyEncryptedTable, EncryptedDynamoDBConfig, EncryptedDynamoDBInstance, } from './types' @@ -16,11 +16,36 @@ import type { * and `bulkDecryptModels` methods that transparently encrypt/decrypt DynamoDB * items according to the provided table schema. * + * Accepts EQL v3 tables (`types.*` domains) and EQL v2 tables + * (`encryptedColumn`/`encryptedField`) alike — the table decides which wire + * format is synthesized on read. + * + * Only equality is meaningful on DynamoDB: an `hm` term is stored alongside the + * ciphertext as `__hmac` and can back a key condition. Ordering and + * free-text terms have no DynamoDB query surface and are not stored, so values + * in those domains remain decryptable but not searchable within DynamoDB. + * * @param config - Configuration containing the `encryptionClient` and optional * logging / error-handling callbacks. * @returns An {@link EncryptedDynamoDBInstance} with encrypt/decrypt operations. * - * @example + * @example EQL v3 + * ```typescript + * import { EncryptionV3, encryptedTable, types } from "@cipherstash/stack/v3" + * import { encryptedDynamoDB } from "@cipherstash/stack/dynamodb" + * + * const users = encryptedTable("users", { + * email: types.TextEq("email"), // equality → queryable via email__hmac + * name: types.Text("name"), // storage only + * }) + * + * const client = await EncryptionV3({ schemas: [users] }) + * const dynamo = encryptedDynamoDB({ encryptionClient: client }) + * + * const encrypted = await dynamo.encryptModel({ email: "a@b.com" }, users) + * ``` + * + * @example EQL v2 (existing deployments) * ```typescript * import { Encryption } from "@cipherstash/stack" * import { encryptedDynamoDB } from "@cipherstash/stack/dynamodb" @@ -32,8 +57,6 @@ import type { * * const client = await Encryption({ schemas: [users] }) * const dynamo = encryptedDynamoDB({ encryptionClient: client }) - * - * const encrypted = await dynamo.encryptModel({ email: "a@b.com" }, users) * ``` */ export function encryptedDynamoDB( @@ -44,7 +67,7 @@ export function encryptedDynamoDB( return { encryptModel>( item: T, - table: EncryptedTable, + table: AnyEncryptedTable, ) { return new EncryptModelOperation( encryptionClient, @@ -56,7 +79,7 @@ export function encryptedDynamoDB( bulkEncryptModels>( items: T[], - table: EncryptedTable, + table: AnyEncryptedTable, ) { return new BulkEncryptModelsOperation( encryptionClient, @@ -68,7 +91,7 @@ export function encryptedDynamoDB( decryptModel>( item: Record, - table: EncryptedTable, + table: AnyEncryptedTable, ) { return new DecryptModelOperation( encryptionClient, @@ -80,7 +103,7 @@ export function encryptedDynamoDB( bulkDecryptModels>( items: Record[], - table: EncryptedTable, + table: AnyEncryptedTable, ) { return new BulkDecryptModelsOperation( encryptionClient, diff --git a/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts b/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts index 110e058d..29daa818 100644 --- a/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts +++ b/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts @@ -1,10 +1,18 @@ import { type Result, withResult } from '@byteslice/result' -import type { EncryptionClient } from '@/encryption' -import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import type { Decrypted, EncryptedValue } from '@/types' import { logger } from '@/utils/logger' -import { handleError, toItemWithEqlPayloads } from '../helpers' -import type { EncryptedDynamoDBError } from '../types' +import { + handleError, + resolveDecryptResult, + throwPreservingCode, + toItemWithEqlPayloads, +} from '../helpers' +import type { + AnyEncryptedTable, + CallableEncryptionClient, + DynamoDBEncryptionClient, + EncryptedDynamoDBError, +} from '../types' import { DynamoDBOperation, type DynamoDBOperationOptions, @@ -13,14 +21,14 @@ import { export class BulkDecryptModelsOperation< T extends Record, > extends DynamoDBOperation[]> { - private encryptionClient: EncryptionClient + private encryptionClient: DynamoDBEncryptionClient private items: Record[] - private table: EncryptedTable + private table: AnyEncryptedTable constructor( - encryptionClient: EncryptionClient, + encryptionClient: DynamoDBEncryptionClient, items: Record[], - table: EncryptedTable, + table: AnyEncryptedTable, options?: DynamoDBOperationOptions, ) { super(options) @@ -39,18 +47,16 @@ export class BulkDecryptModelsOperation< toItemWithEqlPayloads(item, this.table), ) - const decryptResult = await this.encryptionClient - .bulkDecryptModels(itemsWithEqlPayloads as T[]) - .audit(this.getAuditData()) + const client = this.encryptionClient as CallableEncryptionClient + const decryptResult = await resolveDecryptResult[]>( + // The second argument is required by the typed client and ignored by + // the nominal one, which derives the table from the payloads. + client.bulkDecryptModels(itemsWithEqlPayloads, this.table), + this.getAuditData(), + ) if (decryptResult.failure) { - // Create an Error object that preserves the FFI error code - // This is necessary because withResult's ensureError wraps non-Error objects - const error = new Error(decryptResult.failure.message) as Error & { - code?: string - } - error.code = decryptResult.failure.code - throw error + throwPreservingCode(decryptResult.failure) } return decryptResult.data diff --git a/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts b/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts index 7f6cd0b7..bf976569 100644 --- a/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts +++ b/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts @@ -1,9 +1,17 @@ import { type Result, withResult } from '@byteslice/result' -import type { EncryptionClient } from '@/encryption' -import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import { logger } from '@/utils/logger' -import { deepClone, handleError, toEncryptedDynamoItem } from '../helpers' -import type { EncryptedDynamoDBError } from '../types' +import { + deepClone, + handleError, + throwPreservingCode, + toEncryptedDynamoItem, +} from '../helpers' +import type { + AnyEncryptedTable, + CallableEncryptionClient, + DynamoDBEncryptionClient, + EncryptedDynamoDBError, +} from '../types' import { DynamoDBOperation, type DynamoDBOperationOptions, @@ -12,14 +20,14 @@ import { export class BulkEncryptModelsOperation< T extends Record, > extends DynamoDBOperation { - private encryptionClient: EncryptionClient + private encryptionClient: DynamoDBEncryptionClient private items: T[] - private table: EncryptedTable + private table: AnyEncryptedTable constructor( - encryptionClient: EncryptionClient, + encryptionClient: DynamoDBEncryptionClient, items: T[], - table: EncryptedTable, + table: AnyEncryptedTable, options?: DynamoDBOperationOptions, ) { super(options) @@ -32,7 +40,8 @@ export class BulkEncryptModelsOperation< logger.debug(`DynamoDB: bulk encrypting ${this.items.length} models.`) return await withResult( async () => { - const encryptResult = await this.encryptionClient + const client = this.encryptionClient as CallableEncryptionClient + const encryptResult = await client .bulkEncryptModels( this.items.map((item) => deepClone(item)), this.table, @@ -40,13 +49,7 @@ export class BulkEncryptModelsOperation< .audit(this.getAuditData()) if (encryptResult.failure) { - // Create an Error object that preserves the FFI error code - // This is necessary because withResult's ensureError wraps non-Error objects - const error = new Error(encryptResult.failure.message) as Error & { - code?: string - } - error.code = encryptResult.failure.code - throw error + throwPreservingCode(encryptResult.failure) } const data = encryptResult.data.map((item) => deepClone(item)) diff --git a/packages/stack/src/dynamodb/operations/decrypt-model.ts b/packages/stack/src/dynamodb/operations/decrypt-model.ts index dd8702f2..aea746b3 100644 --- a/packages/stack/src/dynamodb/operations/decrypt-model.ts +++ b/packages/stack/src/dynamodb/operations/decrypt-model.ts @@ -1,10 +1,18 @@ import { type Result, withResult } from '@byteslice/result' -import type { EncryptionClient } from '@/encryption' -import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import type { Decrypted, EncryptedValue } from '@/types' import { logger } from '@/utils/logger' -import { handleError, toItemWithEqlPayloads } from '../helpers' -import type { EncryptedDynamoDBError } from '../types' +import { + handleError, + resolveDecryptResult, + throwPreservingCode, + toItemWithEqlPayloads, +} from '../helpers' +import type { + AnyEncryptedTable, + CallableEncryptionClient, + DynamoDBEncryptionClient, + EncryptedDynamoDBError, +} from '../types' import { DynamoDBOperation, type DynamoDBOperationOptions, @@ -13,14 +21,14 @@ import { export class DecryptModelOperation< T extends Record, > extends DynamoDBOperation> { - private encryptionClient: EncryptionClient + private encryptionClient: DynamoDBEncryptionClient private item: Record - private table: EncryptedTable + private table: AnyEncryptedTable constructor( - encryptionClient: EncryptionClient, + encryptionClient: DynamoDBEncryptionClient, item: Record, - table: EncryptedTable, + table: AnyEncryptedTable, options?: DynamoDBOperationOptions, ) { super(options) @@ -37,18 +45,16 @@ export class DecryptModelOperation< async () => { const withEqlPayloads = toItemWithEqlPayloads(this.item, this.table) - const decryptResult = await this.encryptionClient - .decryptModel(withEqlPayloads as T) - .audit(this.getAuditData()) + const client = this.encryptionClient as CallableEncryptionClient + const decryptResult = await resolveDecryptResult>( + // The second argument is required by the typed client and ignored by + // the nominal one, which derives the table from the payloads. + client.decryptModel(withEqlPayloads, this.table), + this.getAuditData(), + ) if (decryptResult.failure) { - // Create an Error object that preserves the FFI error code - // This is necessary because withResult's ensureError wraps non-Error objects - const error = new Error(decryptResult.failure.message) as Error & { - code?: string - } - error.code = decryptResult.failure.code - throw error + throwPreservingCode(decryptResult.failure) } return decryptResult.data diff --git a/packages/stack/src/dynamodb/operations/encrypt-model.ts b/packages/stack/src/dynamodb/operations/encrypt-model.ts index 754420a4..efe36e68 100644 --- a/packages/stack/src/dynamodb/operations/encrypt-model.ts +++ b/packages/stack/src/dynamodb/operations/encrypt-model.ts @@ -1,9 +1,17 @@ import { type Result, withResult } from '@byteslice/result' -import type { EncryptionClient } from '@/encryption' -import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import { logger } from '@/utils/logger' -import { deepClone, handleError, toEncryptedDynamoItem } from '../helpers' -import type { EncryptedDynamoDBError } from '../types' +import { + deepClone, + handleError, + throwPreservingCode, + toEncryptedDynamoItem, +} from '../helpers' +import type { + AnyEncryptedTable, + CallableEncryptionClient, + DynamoDBEncryptionClient, + EncryptedDynamoDBError, +} from '../types' import { DynamoDBOperation, type DynamoDBOperationOptions, @@ -12,14 +20,14 @@ import { export class EncryptModelOperation< T extends Record, > extends DynamoDBOperation { - private encryptionClient: EncryptionClient + private encryptionClient: DynamoDBEncryptionClient private item: T - private table: EncryptedTable + private table: AnyEncryptedTable constructor( - encryptionClient: EncryptionClient, + encryptionClient: DynamoDBEncryptionClient, item: T, - table: EncryptedTable, + table: AnyEncryptedTable, options?: DynamoDBOperationOptions, ) { super(options) @@ -32,18 +40,13 @@ export class EncryptModelOperation< logger.debug('DynamoDB: encrypting model.') return await withResult( async () => { - const encryptResult = await this.encryptionClient + const client = this.encryptionClient as CallableEncryptionClient + const encryptResult = await client .encryptModel(deepClone(this.item), this.table) .audit(this.getAuditData()) if (encryptResult.failure) { - // Create an Error object that preserves the FFI error code - // This is necessary because withResult's ensureError wraps non-Error objects - const error = new Error(encryptResult.failure.message) as Error & { - code?: string - } - error.code = encryptResult.failure.code - throw error + throwPreservingCode(encryptResult.failure) } const data = deepClone(encryptResult.data) diff --git a/packages/stack/src/dynamodb/types.ts b/packages/stack/src/dynamodb/types.ts index a7500d1b..4cdff90a 100644 --- a/packages/stack/src/dynamodb/types.ts +++ b/packages/stack/src/dynamodb/types.ts @@ -1,5 +1,6 @@ import type { ProtectErrorCode } from '@cipherstash/protect-ffi' import type { EncryptionClient } from '@/encryption' +import type { AnyV3Table } from '@/eql/v3' import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import type { EncryptedValue } from '@/types' import type { BulkDecryptModelsOperation } from './operations/bulk-decrypt-models' @@ -7,8 +8,90 @@ import type { BulkEncryptModelsOperation } from './operations/bulk-encrypt-model import type { DecryptModelOperation } from './operations/decrypt-model' import type { EncryptModelOperation } from './operations/encrypt-model' +/** + * A table this adapter accepts: either an EQL v2 table (`encryptedTable` + + * `encryptedColumn`/`encryptedField` from `@cipherstash/stack/schema`) or an + * EQL v3 one (`encryptedTable` + `types.*` from `@cipherstash/stack/eql/v3`). + * + * Both are supported deliberately. DynamoDB shares none of the v2 Postgres + * machinery — there is no EQL extension to install and no migration to run — + * so accepting v3 is purely additive and no existing caller has to change. + */ +export type AnyEncryptedTable = + | EncryptedTable + | AnyV3Table + +/** + * The client capability this adapter consumes, declared structurally so it is + * satisfied by the nominal {@link EncryptionClient} AND by the + * `TypedEncryptionClient` that `EncryptionV3` returns, neither needing a cast. + * Mirrors the approach the Drizzle v3 operators take for the same reason: a + * nominal `TypedEncryptionClient` parameter would reject a client built for + * a narrower schema tuple. + * + * The two clients differ at runtime on the decrypt paths — the nominal client + * returns a chainable operation carrying `.audit()`, the typed wrapper returns + * a plain `Promise>` and takes the table as a second argument. The + * operation classes handle both; see `DecryptModelOperation`. The consequence + * for callers is that **audit metadata on decrypt requires the nominal + * client** — with a client from `EncryptionV3` there is nowhere to put it. + */ +export type DynamoDBEncryptionClient = { + encryptModel(input: never, table: never): unknown + bulkEncryptModels(input: never, table: never): unknown + decryptModel(input: never, table?: never): unknown + bulkDecryptModels(input: never, table?: never): unknown +} + +type ChainableEncryptOperation = { + audit(data: { + metadata?: Record + }): PromiseLike< + | { data: T; failure?: never } + | { data?: never; failure: { message: string; code?: string } } + > +} + +/** + * @internal Callable view of {@link DynamoDBEncryptionClient}. + * + * The public type declares `never` operands so both client shapes satisfy it + * without a cast; a callable signature cannot be written that both a generic + * `EncryptionClient` method and a generic `TypedEncryptionClient` method + * satisfy. The operation classes therefore cast to this shape at the call site + * — the same split the Drizzle v3 operators use. + * + * `decryptModel` is intentionally untyped in its return: the nominal client + * returns a chainable operation, the typed client a plain promise. See + * `resolveDecryptResult`. + */ +export type CallableEncryptionClient = { + encryptModel( + input: Record, + table: AnyEncryptedTable, + ): ChainableEncryptOperation> + bulkEncryptModels( + input: Record[], + table: AnyEncryptedTable, + ): ChainableEncryptOperation[]> + decryptModel( + input: Record, + table?: AnyEncryptedTable, + ): unknown + bulkDecryptModels( + input: Record[], + table?: AnyEncryptedTable, + ): unknown +} + export interface EncryptedDynamoDBConfig { - encryptionClient: EncryptionClient + /** + * Either the nominal client from `Encryption(...)` / `Encryption({ schemas, + * config: { eqlVersion: 3 } })`, or the typed client from `EncryptionV3(...)`. + * For EQL v3 tables the client must be in v3 mode — `EncryptionV3` forces + * this; with `Encryption` you must pass `config: { eqlVersion: 3 }` yourself. + */ + encryptionClient: EncryptionClient | DynamoDBEncryptionClient options?: { logger?: { error: (message: string, error: Error) => void @@ -25,21 +108,21 @@ export interface EncryptedDynamoDBError extends Error { export interface EncryptedDynamoDBInstance { encryptModel>( item: T, - table: EncryptedTable, + table: AnyEncryptedTable, ): EncryptModelOperation bulkEncryptModels>( items: T[], - table: EncryptedTable, + table: AnyEncryptedTable, ): BulkEncryptModelsOperation decryptModel>( item: Record, - table: EncryptedTable, + table: AnyEncryptedTable, ): DecryptModelOperation bulkDecryptModels>( items: Record[], - table: EncryptedTable, + table: AnyEncryptedTable, ): BulkDecryptModelsOperation } diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index 3c8c26b1..ee63eb43 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -1,6 +1,6 @@ --- name: stash-dynamodb -description: Integrate CipherStash encryption with Amazon DynamoDB using @cipherstash/stack/dynamodb. Covers the encryptedDynamoDB helper for encrypting items before PutItem and decrypting after GetItem, bulk encrypt/decrypt for BatchWrite and BatchGet, querying with encrypted partition and sort keys via HMAC attributes, nested object encryption, audit logging, and the DynamoDB attribute naming conventions (__source/__hmac). Use when adding encryption to a DynamoDB project, encrypting items before writes, decrypting items after reads, or querying encrypted DynamoDB attributes. +description: Integrate CipherStash encryption with Amazon DynamoDB using @cipherstash/stack/dynamodb. Covers the encryptedDynamoDB helper for encrypting items before PutItem and decrypting after GetItem with EQL v3 (types.* domains) or EQL v2 schemas, bulk encrypt/decrypt for BatchWrite and BatchGet, querying with encrypted partition and sort keys via HMAC attributes, nested object encryption, audit logging, and the DynamoDB attribute naming conventions (__source/__hmac). Use when adding encryption to a DynamoDB project, encrypting items before writes, decrypting items after reads, or querying encrypted DynamoDB attributes. --- # CipherStash Stack - DynamoDB Integration @@ -36,68 +36,111 @@ CipherStash encrypts each attribute into two DynamoDB attributes: | Original Attribute | Stored As | Purpose | |---|---|---| | `email` | `email__source` | Encrypted ciphertext | -| `email` | `email__hmac` | HMAC for equality lookups (only if `.equality()` index is set) | +| `email` | `email__hmac` | HMAC for equality lookups (only for equality-capable columns) | Non-encrypted attributes pass through unchanged. On decryption, the `__source` and `__hmac` attributes are recombined back into the original attribute name with the plaintext value. +**Only equality is usable on DynamoDB.** Ordering terms and free-text bloom filters have no DynamoDB query surface, so they are not stored. A column in an ordering or free-text domain still encrypts and decrypts correctly — it just cannot back a key condition. + +## Choosing a Schema Version + +`encryptedDynamoDB` accepts both EQL v3 and EQL v2 tables. **Use EQL v3 for new projects.** + +| | EQL v3 (recommended) | EQL v2 (existing deployments) | +|---|---|---| +| Import | `@cipherstash/stack/v3` | `@cipherstash/stack` + `@cipherstash/stack/schema` | +| Schema | `encryptedTable` + `types.*` | `encryptedTable` + `encryptedColumn` | +| Client | `EncryptionV3({ schemas })` | `Encryption({ schemas })` | +| Nested fields | **Not supported** | `encryptedField` | + +There is no data migration between them: DynamoDB has no EQL extension to install and no schema to alter. But the two write **different wire formats**, so a table populated under v2 cannot be read back with a v3 schema, or vice versa. Pick one per table and stay on it. + +If you need nested object encryption, you must use EQL v2 — v3 has no `encryptedField` equivalent. + ## Rolling Encryption Out to Production DynamoDB encryption is **single-deploy**. There is no rollout/cutover split — unlike the Postgres path, DynamoDB has no row-level rename swap and no shared-state proxy. The application owns every write, so adding encryption is an application-side change that ships in one PR: 1. Declare the encrypted schema (see Setup below). -2. Wrap your DynamoDB client with `encryptedDynamoDB` (or call `encryptItem` / `decryptItem` directly at write/read sites). +2. Build an `encryptedDynamoDB` helper and call `encryptModel` / `decryptModel` at your write and read sites. 3. Ship the change. -For tables with **existing populated items**, the `__source` and `__hmac` attributes are added by the next write that touches each item. If you need every existing item encrypted at once (e.g. because a query uses `email__hmac` and would miss legacy items), run a one-shot script that reads every item, calls `encryptItem`, and writes it back. Idempotent: re-running an already-encrypted item is a no-op as long as the schema hasn't changed. +For tables with **existing populated items**, the `__source` and `__hmac` attributes are added by the next write that touches each item. If you need every existing item encrypted at once (e.g. because a query uses `email__hmac` and would miss legacy items), run a one-shot script that reads every item, calls `encryptModel`, and writes it back. Idempotent: re-running an already-encrypted item is a no-op as long as the schema hasn't changed. > **Where am I?** Run `stash status` (or `bunx`/`pnpm dlx`/`yarn dlx` per your runner) for a project-wide view across both Postgres and DynamoDB integrations. DynamoDB columns surface in the quest log as already-complete since there is no staged lifecycle to track. ## Setup -### 1. Define Encrypted Schema +### 1. Define Encrypted Schema (EQL v3) + +Each `types.*` factory is a concrete domain with fixed query capabilities. There are no chainable index methods — the type *is* the capability. ```typescript -import { encryptedTable, encryptedColumn, encryptedField } from "@cipherstash/stack/schema" +import { encryptedTable, types } from "@cipherstash/stack/v3" const users = encryptedTable("users", { - email: encryptedColumn("email").equality(), // searchable via HMAC - name: encryptedColumn("name"), // encrypt-only, no search - phone: encryptedColumn("phone"), // encrypt-only - metadata: encryptedColumn("metadata").dataType("json"), // encrypt-only JSON (use .searchableJson() for queryable JSON) + email: types.TextEq("email"), // equality -> queryable via email__hmac + name: types.Text("name"), // storage only + phone: types.Text("phone"), // storage only + age: types.IntegerOrd("age"), // decryptable, NOT queryable on DynamoDB + metadata: types.Json("metadata"), // JSON document }) ``` -> **Note:** `encryptedColumn` also supports `.orderAndRange()`, `.freeTextSearch()`, and `.searchableJson()` index methods, but only `.equality()` produces HMAC values usable for DynamoDB key condition queries. +Which domains give you a `__hmac` attribute you can query on: -Nested objects are supported with `encryptedField`: +| Domain family | `__hmac` written? | Notes | +|---|---|---| +| `types.TextEq`, `IntegerEq`, `BigintEq`, `DateEq`, `TimestampEq`, `NumericEq`, `RealEq`, `DoubleEq`, `SmallintEq` | Yes | The equality domains — use these for anything you query | +| `types.TextOrd`, `TextOrdOre`, `TextSearch` | Yes | Text equality is HMAC-based even on ordering/search domains | +| `types.IntegerOrd`, `DateOrd`, `TimestampOrd`, and the other non-text `*Ord`/`*OrdOre` | **No** | Equality resolves through an ordering term in Postgres, which DynamoDB cannot use | +| `types.Text`, `Integer`, `Boolean`, and the other bare domains | No | Storage only | +| `types.Json` | No | Index terms live inside the ste_vec array; not splittable into an attribute | -```typescript -const users = encryptedTable("users", { - email: encryptedColumn("email").equality(), - profile: { - ssn: encryptedField("profile.ssn"), - address: { - street: encryptedField("profile.address.street"), - }, - }, -}) -``` +> **Rule of thumb:** if an attribute will appear in a `KeyConditionExpression`, declare it with an `*Eq` domain. ### 2. Initialize Clients ```typescript import { DynamoDBClient } from "@aws-sdk/client-dynamodb" import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb" -import { Encryption } from "@cipherstash/stack" +import { EncryptionV3 } from "@cipherstash/stack/v3" import { encryptedDynamoDB } from "@cipherstash/stack/dynamodb" const dynamoClient = new DynamoDBClient({ region: "us-east-1" }) const docClient = DynamoDBDocumentClient.from(dynamoClient) +const encryptionClient = await EncryptionV3({ schemas: [users] }) +const dynamo = encryptedDynamoDB({ encryptionClient }) +``` + +> **Audit metadata on decrypt:** the client from `EncryptionV3` has no audit +> surface on its decrypt methods, so `.audit()` on `decryptModel` / +> `bulkDecryptModels` resolves normally but the metadata is not recorded. If you +> need audited decrypts, build the client with `Encryption({ schemas: [users], +> config: { eqlVersion: 3 } })` instead — same v3 wire format, chainable decrypt. + +### EQL v2 Schema (existing deployments) + +```typescript +import { encryptedTable, encryptedColumn, encryptedField } from "@cipherstash/stack/schema" +import { Encryption } from "@cipherstash/stack" + +const users = encryptedTable("users", { + email: encryptedColumn("email").equality(), // searchable via HMAC + name: encryptedColumn("name"), // encrypt-only, no search + metadata: encryptedColumn("metadata").dataType("json"), + profile: { // nested objects: v2 only + ssn: encryptedField("profile.ssn"), + }, +}) + const encryptionClient = await Encryption({ schemas: [users] }) const dynamo = encryptedDynamoDB({ encryptionClient }) ``` +> **Note:** `encryptedColumn` also supports `.orderAndRange()`, `.freeTextSearch()`, and `.searchableJson()` index methods, but only `.equality()` produces HMAC values usable for DynamoDB key condition queries. + ### Optional: Logger and Error Handler ```typescript @@ -228,19 +271,19 @@ When an encrypted attribute is the partition key (e.g., `email__hmac`): ```typescript import { QueryCommand } from "@aws-sdk/lib-dynamodb" -// 1. Encrypt the search value to get the HMAC -const queryResult = await encryptionClient.encryptQuery([{ - value: "alice@example.com", - column: users.email, +// 1. Encrypt the search value to get the HMAC. +// On an EQL v3 equality domain this mints the bare term — `{ v, i, hm }` +// with no ciphertext — so `hm` is used directly. +const queryResult = await encryptionClient.encryptQuery("alice@example.com", { table: users, - queryType: "equality", -}]) + column: users.email, +}) if (queryResult.failure) { throw new Error(`Query encryption failed: ${queryResult.failure.message}`) } -const emailHmac = queryResult.data[0]?.hm +const emailHmac = queryResult.data.hm // 2. Use the HMAC in a DynamoDB query const result = await docClient.send(new QueryCommand({ @@ -316,7 +359,7 @@ For each encrypted field with an equality index, two attributes are stored: - `{field}__source` - The encrypted ciphertext (binary/string) - `{field}__hmac` - Deterministic HMAC for equality lookups -Fields without `.equality()` only get `__source` (no HMAC, so they can't be queried). +Fields without equality capability only get `__source` (no HMAC, so they can't be queried) — that means EQL v2 columns without `.equality()`, and EQL v3 columns outside the domains listed in the table above. ### Key Schema Design @@ -361,7 +404,8 @@ if (result.failure) { import { encryptedDynamoDB } from "@cipherstash/stack/dynamodb" const dynamo = encryptedDynamoDB({ - encryptionClient, // EncryptionClient instance + // From EncryptionV3(...) or Encryption(...) + encryptionClient, options: { // optional logger: { error: (message, error) => void }, errorHandler: (error) => void, @@ -371,56 +415,59 @@ const dynamo = encryptedDynamoDB({ ### Instance Methods +`table` is either an EQL v3 table (`types.*` domains) or an EQL v2 one. + | Method | Signature | Returns | |---|---|---| -| `encryptModel` | `(item: T, table: EncryptedTable)` | `EncryptModelOperation` | -| `bulkEncryptModels` | `(items: T[], table: EncryptedTable)` | `BulkEncryptModelsOperation` | -| `decryptModel` | `(item: Record, table: EncryptedTable)` | `DecryptModelOperation` (resolves to `Decrypted`) | -| `bulkDecryptModels` | `(items: Record[], table: EncryptedTable)` | `BulkDecryptModelsOperation` (resolves to `Decrypted[]`) | +| `encryptModel` | `(item: T, table)` | `EncryptModelOperation` | +| `bulkEncryptModels` | `(items: T[], table)` | `BulkEncryptModelsOperation` | +| `decryptModel` | `(item: Record, table)` | `DecryptModelOperation` (resolves to `Decrypted`) | +| `bulkDecryptModels` | `(items: Record[], table)` | `BulkDecryptModelsOperation` (resolves to `Decrypted[]`) | -All operations are thenable (awaitable) and support `.audit({ metadata })` chaining. +All operations are thenable (awaitable) and support `.audit({ metadata })` chaining. See the note in Setup about decrypt-side audit metadata and the `EncryptionV3` client. ### Querying Encrypted Attributes Use the encryption client directly (not the DynamoDB helper): ```typescript -// Single value form (recommended for DynamoDB lookups): +// EQL v3 — the domain fixes the query type, so no `queryType` is needed: const result = await encryptionClient.encryptQuery( "search-value", - { column: schema.fieldName, table: schema, queryType: "equality" } + { table: users, column: users.email } +) +const hmac = result.data?.hm // Use this in DynamoDB key conditions + +// EQL v2 — pass queryType explicitly: +const v2Result = await encryptionClient.encryptQuery( + "search-value", + { table: users, column: users.email, queryType: "equality" } ) -const hmac = result.data?.hm - -// Batch array form: -const batchResult = await encryptionClient.encryptQuery([{ - value: "search-value", - column: schema.fieldName, - table: schema, - queryType: "equality", -}]) -const hmac = batchResult.data[0]?.hm // Use this in DynamoDB key conditions +const v2Hmac = v2Result.data?.hm ``` +> On a `types.TextSearch` column `encryptQuery` returns `hm` alongside ordering +> and bloom-filter terms regardless of `queryType`. Only `hm` is meaningful for +> DynamoDB — a free-text query cannot be expressed as a key condition. + ## Complete Example ```typescript import { DynamoDBClient } from "@aws-sdk/client-dynamodb" import { DynamoDBDocumentClient, PutCommand, GetCommand, QueryCommand } from "@aws-sdk/lib-dynamodb" -import { Encryption } from "@cipherstash/stack" +import { EncryptionV3, encryptedTable, types } from "@cipherstash/stack/v3" import { encryptedDynamoDB } from "@cipherstash/stack/dynamodb" -import { encryptedTable, encryptedColumn } from "@cipherstash/stack/schema" // Schema const users = encryptedTable("users", { - email: encryptedColumn("email").equality(), - name: encryptedColumn("name"), + email: types.TextEq("email"), + name: types.Text("name"), }) // Clients const dynamoClient = new DynamoDBClient({ region: "us-east-1" }) const docClient = DynamoDBDocumentClient.from(dynamoClient) -const encryptionClient = await Encryption({ schemas: [users] }) +const encryptionClient = await EncryptionV3({ schemas: [users] }) const dynamo = encryptedDynamoDB({ encryptionClient }) // Write @@ -441,13 +488,11 @@ if (!decResult.failure) { } // Query by encrypted email (via HMAC) -const queryEnc = await encryptionClient.encryptQuery([{ - value: "alice@example.com", - column: users.email, +const queryEnc = await encryptionClient.encryptQuery("alice@example.com", { table: users, - queryType: "equality", -}]) -const hmac = queryEnc.data[0]?.hm + column: users.email, +}) +const hmac = queryEnc.data?.hm const queryResult = await docClient.send(new QueryCommand({ TableName: "Users", From 84b88651cc70f0ed81efd3de9dab855845dd095b Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 19:56:08 +1000 Subject: [PATCH 03/20] fix(stack): type the DynamoDB storage payload structurally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write-path rewrite dropped the `k === 'ct'` check that was narrowing `EncryptedValue` to `EncryptedScalar`, so reading `c`/`hm` off the union no longer compiled — `EncryptedValue` is v2-only (`EncryptedScalar | EncryptedSteVec`) and has no member describing an untagged v3 scalar. Introduce a local `StoredEqlPayload` stating the four keys the mapping actually touches. Caught by running tsc directly: `npx tsc` resolves to a different package in this workspace and prints a banner instead of compiling, so it silently reported no errors. --- packages/stack/src/dynamodb/helpers.ts | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index c2252fa3..6ca983bc 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -157,6 +157,27 @@ export function deepClone(obj: T): T { ) } +/** + * The storage payloads this adapter splits into DynamoDB attributes, across + * both wire versions. + * + * `EncryptedValue` from `@/types` cannot describe these: its union is + * `EncryptedScalar | EncryptedSteVec`, both v2-only, so it has no member + * matching an untagged v3 scalar and reading `c`/`hm` off it does not compile + * once the `k: 'ct'` narrowing is gone. The mapping only ever touches four + * keys, so state them structurally. + */ +type StoredEqlPayload = { + /** Absent on v3 scalars; `'ct'` on v2 scalars; `'sv'` on JSON in both. */ + k?: 'ct' | 'sv' + /** Scalar ciphertext. */ + c?: unknown + /** Deterministic equality term — the only one DynamoDB can query. */ + hm?: unknown + /** ste_vec entries for a JSON document. */ + sv?: unknown +} + export function toEncryptedDynamoItem( encrypted: Record, encryptedAttrs: string[], @@ -177,7 +198,7 @@ export function toEncryptedDynamoItem( typeof attrValue === 'object' && 'c' in (attrValue as object)) ) { - const encryptPayload = attrValue as EncryptedValue + const encryptPayload = attrValue as StoredEqlPayload // A JSON document, in either wire version, keeps its `k: 'sv'` tag. Its // index terms live *inside* the `sv` entries, so the whole array is From 35b155fcce035efaf2eacdcbd6f39d8325129cdf Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 19:56:47 +1000 Subject: [PATCH 04/20] docs(skills): sharpen the v2/v3 nested-object distinction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous wording — "nested object encryption requires v2" — was too absolute. v3 encrypts a nested object wholesale via `types.Json`; what it cannot do is v2's per-leaf `encryptedField`, which encrypts selected leaves in place and leaves siblings plaintext. Show both shapes and when to pick each. --- skills/stash-dynamodb/SKILL.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index ee63eb43..066555d4 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -55,7 +55,28 @@ Non-encrypted attributes pass through unchanged. On decryption, the `__source` a There is no data migration between them: DynamoDB has no EQL extension to install and no schema to alter. But the two write **different wire formats**, so a table populated under v2 cannot be read back with a v3 schema, or vice versa. Pick one per table and stay on it. -If you need nested object encryption, you must use EQL v2 — v3 has no `encryptedField` equivalent. +**Nested objects work differently in each version, and this is the main reason to stay on v2.** + +EQL v2's `encryptedField` encrypts *selected leaves in place*. The item keeps its shape and unlisted siblings stay plaintext: + +```jsonc +// schema: profile: { ssn: encryptedField("profile.ssn") } +{ "profile": { "ssn__source": "", "city": "Sydney" } } +``` + +EQL v3 has no `encryptedField` — a nested group is a compile error, since a v3 column map holds only `types.*` domains. Its equivalent is `types.Json`, which encrypts the **whole subtree as a single value**: + +```jsonc +// schema: profile: types.Json("profile") +{ "profile__source": [ /* ste_vec entries */ ] } +``` + +Choose accordingly: + +| You want | Use | +|---|---| +| The whole object encrypted as one value | `types.Json` (v3) | +| Some leaves encrypted, siblings plaintext, structure preserved | `encryptedField` (v2 only) | ## Rolling Encryption Out to Production From ffa7e3a9ef09fdbc825a4425575b4172aa580ce4 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 20:07:23 +1000 Subject: [PATCH 05/20] docs: nested fields in v3 are deferred, not decided against MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous wording implied a closed design decision. The code and the original scoping spec both frame it as deferral — `columns.ts`: "Nested fields are deferred to later increments"; the v3 schema spec lists nested `encryptedField` under increment-1 non-goals; the table type helpers carry a comment marking the flat shape as a future extension point. No ADR, issue, or commit anywhere records a rationale for dropping it, so do not imply one. --- .changeset/dynamodb-eql-v3.md | 5 +++-- AGENTS.md | 2 +- skills/stash-dynamodb/SKILL.md | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md index bc6335ec..da80de94 100644 --- a/.changeset/dynamodb-eql-v3.md +++ b/.changeset/dynamodb-eql-v3.md @@ -28,8 +28,9 @@ Notes on capability: that mint an `hm` term — the `*Eq` family, plus `TextOrd`/`TextOrdOre`/ `TextSearch`. Ordering and bloom-filter terms have no DynamoDB query surface and are not stored, so those columns remain decryptable but not queryable. -- EQL v3 has no `encryptedField` equivalent, so nested object encryption - requires an EQL v2 table. +- EQL v3 has no `encryptedField` equivalent yet (deferred, not decided + against), so per-leaf nested object encryption still requires an EQL v2 + table. v3 can encrypt a whole subtree with `types.Json`. - Audit metadata on `decryptModel` / `bulkDecryptModels` requires the nominal client; the `EncryptionV3` client has no audit surface on decrypt. diff --git a/AGENTS.md b/AGENTS.md index 85ba6444..a200f1ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,7 +147,7 @@ Three rules to remember when editing CI or pnpm config: ## Key Concepts and APIs - **Initialization**: `EncryptionV3({ schemas })` returns the typed EQL v3 client. (`Encryption({ schemas })` is the legacy v2 client.) Provide at least one `encryptedTable`. -- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments. DynamoDB accepts both v3 and v2 tables; only v2 supports nested fields via `encryptedField`.) +- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments. DynamoDB accepts both v3 and v2 tables; per-leaf nested fields via `encryptedField` are v2-only, deferred in v3 — see `packages/stack/src/eql/v3/columns.ts`.) - **Operations** (all return Result-like objects and support chaining `.withLockContext(lockContext)` and `.audit()` when applicable): - `encrypt(plaintext, { table, column })` - `decrypt(encryptedPayload)` diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index 066555d4..1dc4d5e6 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -51,7 +51,7 @@ Non-encrypted attributes pass through unchanged. On decryption, the `__source` a | Import | `@cipherstash/stack/v3` | `@cipherstash/stack` + `@cipherstash/stack/schema` | | Schema | `encryptedTable` + `types.*` | `encryptedTable` + `encryptedColumn` | | Client | `EncryptionV3({ schemas })` | `Encryption({ schemas })` | -| Nested fields | **Not supported** | `encryptedField` | +| Nested fields | **Not yet** (deferred) | `encryptedField` | There is no data migration between them: DynamoDB has no EQL extension to install and no schema to alter. But the two write **different wire formats**, so a table populated under v2 cannot be read back with a v3 schema, or vice versa. Pick one per table and stay on it. @@ -64,7 +64,9 @@ EQL v2's `encryptedField` encrypts *selected leaves in place*. The item keeps it { "profile": { "ssn__source": "", "city": "Sydney" } } ``` -EQL v3 has no `encryptedField` — a nested group is a compile error, since a v3 column map holds only `types.*` domains. Its equivalent is `types.Json`, which encrypts the **whole subtree as a single value**: +EQL v3 has no `encryptedField` yet — a nested group is a compile error, since a v3 column map holds only `types.*` domains. This is deferred work rather than a closed design decision (`packages/stack/src/eql/v3/columns.ts`: "Nested fields are deferred to later increments"), so check the current release before assuming it is still missing. + +The nearest v3 equivalent today is `types.Json`, which encrypts the **whole subtree as a single value**: ```jsonc // schema: profile: types.Json("profile") From 440caa6ef2f74cad115030540a9aff1cd8e3259c Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 20:28:06 +1000 Subject: [PATCH 06/20] =?UTF-8?q?docs:=20correct=20the=20v3=20nested-field?= =?UTF-8?q?=20framing=20again=20=E2=80=94=20it=20is=20a=20regression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified empirically: nested encrypt AND decrypt round-trip correctly under EQL v3 today with no code change, provided the column path is dotted. encryptedTable('t', { 'profile.ssn': types.TextEq('profile.ssn') }) encryptModel({ profile: { ssn: 'secret' } }) -> encrypts decryptModel(...) -> { profile: { ssn: 'secret' } } So it is not deferred work and not a design decision. The nested model walk in encryption/helpers/model-helpers.ts is generic and already shared. v3 fails only because resolveEncryptColumnMap takes the buildColumnKeyMap() branch, which returns flat JS property names, while the walk builds dotted model paths. v2 takes the other branch and gets dotted DB column names. What is genuinely missing is the authoring form: EncryptedV3TableColumn does not admit nested groups, and build()/buildColumnKeyMap() do not flatten them. --- .changeset/dynamodb-eql-v3.md | 6 +++--- AGENTS.md | 2 +- skills/stash-dynamodb/SKILL.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md index da80de94..c2a8f4f8 100644 --- a/.changeset/dynamodb-eql-v3.md +++ b/.changeset/dynamodb-eql-v3.md @@ -28,9 +28,9 @@ Notes on capability: that mint an `hm` term — the `*Eq` family, plus `TextOrd`/`TextOrdOre`/ `TextSearch`. Ordering and bloom-filter terms have no DynamoDB query surface and are not stored, so those columns remain decryptable but not queryable. -- EQL v3 has no `encryptedField` equivalent yet (deferred, not decided - against), so per-leaf nested object encryption still requires an EQL v2 - table. v3 can encrypt a whole subtree with `types.Json`. +- EQL v3 has no `encryptedField` authoring form, so per-leaf nested object + encryption currently requires an EQL v2 table. v3 can encrypt a whole + subtree with `types.Json`. - Audit metadata on `decryptModel` / `bulkDecryptModels` requires the nominal client; the `EncryptionV3` client has no audit surface on decrypt. diff --git a/AGENTS.md b/AGENTS.md index a200f1ef..aa9903ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,7 +147,7 @@ Three rules to remember when editing CI or pnpm config: ## Key Concepts and APIs - **Initialization**: `EncryptionV3({ schemas })` returns the typed EQL v3 client. (`Encryption({ schemas })` is the legacy v2 client.) Provide at least one `encryptedTable`. -- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments. DynamoDB accepts both v3 and v2 tables; per-leaf nested fields via `encryptedField` are v2-only, deferred in v3 — see `packages/stack/src/eql/v3/columns.ts`.) +- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments. DynamoDB accepts both v3 and v2 tables; per-leaf nested fields via `encryptedField` are v2-only for now — v3's model walk supports nesting, but `EncryptedV3TableColumn` has no nested authoring form.) - **Operations** (all return Result-like objects and support chaining `.withLockContext(lockContext)` and `.audit()` when applicable): - `encrypt(plaintext, { table, column })` - `decrypt(encryptedPayload)` diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index 1dc4d5e6..53e4edce 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -64,9 +64,9 @@ EQL v2's `encryptedField` encrypts *selected leaves in place*. The item keeps it { "profile": { "ssn__source": "", "city": "Sydney" } } ``` -EQL v3 has no `encryptedField` yet — a nested group is a compile error, since a v3 column map holds only `types.*` domains. This is deferred work rather than a closed design decision (`packages/stack/src/eql/v3/columns.ts`: "Nested fields are deferred to later increments"), so check the current release before assuming it is still missing. +EQL v3 has no `encryptedField` authoring form — a nested group is a compile error, since a v3 column map holds only `types.*` domains. -The nearest v3 equivalent today is `types.Json`, which encrypts the **whole subtree as a single value**: +The nearest v3 equivalent is `types.Json`, which encrypts the **whole subtree as a single value**: ```jsonc // schema: profile: types.Json("profile") From 02e75f5c2b6deb9905b0d0ca3be023b03a89049b Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 20:36:06 +1000 Subject: [PATCH 07/20] feat(stack): support nested v3 attributes in encryptedDynamoDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DynamoDB items are natively nested, so v3 support for this adapter is not complete without them. It turns out nothing needed building: the nested model walk in encryption/helpers/model-helpers.ts is shared with v2 and matches on dotted paths, so a v3 column declared with a dotted path works end to end today. encryptedTable('users', { 'profile.ssn': types.TextEq('profile.ssn') }) encryptModel({ pk, profile: { ssn, city } }) -> { pk, profile: { ssn__source, ssn__hmac, city } } Nested equality attributes keep their __hmac, so a key condition on profile.ssn__hmac works, and plaintext siblings are untouched. Also fixes the nested identifier. The read path rebuilt `i.c` from the leaf attribute name (`ssn`) rather than the registered column (`profile.ssn`). That only ever worked because the FFI treats `i` as a detection key and never validates it against the ciphertext — a latent trap that would bite if that were ever tightened. It now prefers the registered dotted path and falls back to the leaf, so both v2 authoring conventions — `encryptedField('example.protected')` and `encryptedField('amount')` under a group — rebuild an identifier matching how the column was declared. The one characterisation test pinning the old leaf-name behaviour is updated with the reasoning; every live round-trip test passed unchanged, which is what shows the identifier change is safe. What still has no v3 equivalent is only the nested-group *authoring* form. Adding it means widening EncryptedV3TableColumn, which breaks six flat iterations of columnBuilders in prisma-next and stack-supabase — ergonomics, not capability. --- .changeset/dynamodb-eql-v3.md | 8 +- AGENTS.md | 2 +- .../dynamodb/encrypted-dynamodb-v3.test.ts | 127 ++++++++++++++++++ .../stack/__tests__/dynamodb/helpers.test.ts | 29 +++- packages/stack/src/dynamodb/helpers.ts | 40 ++++-- skills/stash-dynamodb/SKILL.md | 43 +++--- 6 files changed, 215 insertions(+), 34 deletions(-) diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md index c2a8f4f8..67d9a7c2 100644 --- a/.changeset/dynamodb-eql-v3.md +++ b/.changeset/dynamodb-eql-v3.md @@ -28,9 +28,11 @@ Notes on capability: that mint an `hm` term — the `*Eq` family, plus `TextOrd`/`TextOrdOre`/ `TextSearch`. Ordering and bloom-filter terms have no DynamoDB query surface and are not stored, so those columns remain decryptable but not queryable. -- EQL v3 has no `encryptedField` authoring form, so per-leaf nested object - encryption currently requires an EQL v2 table. v3 can encrypt a whole - subtree with `types.Json`. +- Nested attributes are supported in v3. There is no nested-group authoring + form (that is a compile error), so declare the column flat with a dotted + path — `{ 'profile.ssn': types.TextEq('profile.ssn') }`. The model is + matched by dotted path, so `{ profile: { ssn } }` resolves, and the nested + attribute keeps its `__hmac` for key conditions. - Audit metadata on `decryptModel` / `bulkDecryptModels` requires the nominal client; the `EncryptionV3` client has no audit surface on decrypt. diff --git a/AGENTS.md b/AGENTS.md index aa9903ed..41b11a82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,7 +147,7 @@ Three rules to remember when editing CI or pnpm config: ## Key Concepts and APIs - **Initialization**: `EncryptionV3({ schemas })` returns the typed EQL v3 client. (`Encryption({ schemas })` is the legacy v2 client.) Provide at least one `encryptedTable`. -- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments. DynamoDB accepts both v3 and v2 tables; per-leaf nested fields via `encryptedField` are v2-only for now — v3's model walk supports nesting, but `EncryptedV3TableColumn` has no nested authoring form.) +- **Schema (EQL v3, the documented approach)**: Define tables/columns with `encryptedTable` and the `types.*` concrete-domain factories from `@cipherstash/stack/eql/v3` (`types.TextSearch`, `types.IntegerOrd`, `types.Json`, …) — each domain's query capabilities are fixed by its type; there are no chainable capability tuners. Build the client with `EncryptionV3` from `@cipherstash/stack/v3`. (Legacy EQL v2 — `Encryption` + `encryptedColumn(...).equality().freeTextSearch().orderAndRange().searchableJson()` from `@cipherstash/stack/schema` — remains for existing deployments. DynamoDB accepts both v3 and v2 tables; nested fields work in both — v2 via `encryptedField` groups, v3 via a flat dotted column path (`'profile.ssn': types.TextEq(...)`), since `EncryptedV3TableColumn` admits no nested groups.) - **Operations** (all return Result-like objects and support chaining `.withLockContext(lockContext)` and `.audit()` when applicable): - `encrypt(plaintext, { table, column })` - `decrypt(encryptedPayload)` diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index 2cbfb8d7..b9b7f1ef 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -17,6 +17,7 @@ import 'dotenv/config' import { beforeAll, describe, expect, it } from 'vitest' import { encryptedDynamoDB } from '@/dynamodb' +import { toItemWithEqlPayloads } from '@/dynamodb/helpers' import type { EncryptedDynamoDBInstance } from '@/dynamodb/types' import type { EncryptionClient } from '@/encryption' import { EncryptionV3 } from '@/encryption/v3' @@ -241,6 +242,132 @@ describe('bulk operations with a v3 table', () => { }) }) +describe('nested attributes with a v3 table', () => { + // EQL v3 has no nested-object *authoring* form — a nested group is a compile + // error. But a dotted column path is a supported column name, and the model + // walk in `encryption/helpers/model-helpers.ts` is shared with v2 and matches + // on dotted paths. So nested DynamoDB attributes work in v3 today, declared + // flat. This matters for DynamoDB specifically, where items are natively + // nested documents. + const nested = encryptedTable('users_v3_nested', { + 'profile.ssn': types.TextEq('profile.ssn'), + 'profile.note': types.Text('profile.note'), + }) + + type NestedUser = { + pk: string + profile?: { ssn?: string; note?: string; city?: string } + } + + let nestedDynamo: EncryptedDynamoDBInstance + let nestedClient: EncryptionClient + + beforeAll(async () => { + nestedClient = await Encryption({ + schemas: [nested] as never, + config: { eqlVersion: 3 }, + }) + nestedDynamo = encryptedDynamoDB({ encryptionClient: nestedClient }) + }) + + it('splits nested attributes in place, leaving plaintext siblings alone', async () => { + const result = await nestedDynamo.encryptModel( + { + pk: 'u#1', + profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, + }, + nested, + ) + + if (result.failure) throw new Error(result.failure.message) + + const profile = result.data.profile as Record + expect(Object.keys(profile).sort()).toEqual([ + 'city', + 'note__source', + 'ssn__hmac', + 'ssn__source', + ]) + expect(profile.city).toBe('Sydney') + }) + + it('round-trips a nested item exactly', async () => { + const original: NestedUser = { + pk: 'u#2', + profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, + } + + const encrypted = await nestedDynamo.encryptModel( + original, + nested, + ) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await nestedDynamo.decryptModel( + encrypted.data, + nested, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }) + + it('rebuilds the nested identifier from its registered dotted name', async () => { + const encrypted = await nestedDynamo.encryptModel( + { pk: 'u#3', profile: { ssn: '123-45-6789' } }, + nested, + ) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const rebuilt = toItemWithEqlPayloads(encrypted.data, nested) + const payload = (rebuilt.profile as Record) + .ssn + + expect(payload.i.c).toBe('profile.ssn') + }) + + it('makes a nested equality attribute queryable by __hmac', async () => { + const ssn = '999-88-7777' + + const stored = await nestedDynamo.encryptModel( + { pk: 'u#4', profile: { ssn } }, + nested, + ) + if (stored.failure) throw new Error(stored.failure.message) + + const term = await nestedClient.encryptQuery(ssn, { + table: nested as never, + column: nested['profile.ssn'] as never, + }) + if (term.failure) throw new Error(term.failure.message) + + const profile = stored.data.profile as Record + // A key condition on `profile.ssn__hmac` matches what was written. + expect((term.data as { hm: string }).hm).toBe(profile.ssn__hmac) + }) + + it('bulk round-trips nested items', async () => { + const items: NestedUser[] = [ + { pk: 'u#5', profile: { ssn: 'a', city: 'Sydney' } }, + { pk: 'u#6', profile: { note: 'b' } }, + ] + + const encrypted = await nestedDynamo.bulkEncryptModels( + items, + nested, + ) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await nestedDynamo.bulkDecryptModels( + encrypted.data, + nested, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(items) + }) +}) + describe('the __hmac key-condition path with a v3 table', () => { it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { const email = 'heidi@example.com' diff --git a/packages/stack/__tests__/dynamodb/helpers.test.ts b/packages/stack/__tests__/dynamodb/helpers.test.ts index 9724941d..3c664237 100644 --- a/packages/stack/__tests__/dynamodb/helpers.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers.test.ts @@ -195,7 +195,7 @@ describe('toItemWithEqlPayloads (read path)', () => { }) }) - it('rebuilds nested payloads, identifying them by the leaf attribute name', () => { + it('rebuilds nested payloads, identifying them by their registered dotted name', () => { const result = toItemWithEqlPayloads( { example: { protected__source: 'nested-ct', notProtected: 'plain' } }, users, @@ -204,9 +204,11 @@ describe('toItemWithEqlPayloads (read path)', () => { expect(result).toEqual({ example: { protected: { - // NOTE: the identifier uses the leaf name, not the dotted schema key - // (`example.protected`) the column is registered under. - i: { c: 'protected', t: 'users' }, + // The column is registered as `example.protected`, so that is what + // the rebuilt identifier carries. This originally emitted the bare + // leaf (`protected`), which only worked because the FFI treats `i` as + // a detection key and never validates it against the ciphertext. + i: { c: 'example.protected', t: 'users' }, v: 2, k: 'ct', c: 'nested-ct', @@ -216,6 +218,25 @@ describe('toItemWithEqlPayloads (read path)', () => { }) }) + it('falls back to the leaf name when only the leaf is registered', () => { + // `encryptedField('amount')` under a `details` group — the convention the + // schema docs show — registers `amount`, not `details.amount`. + const orders = encryptedTable('orders', { + details: { amount: encryptedField('amount') }, + }) + + const result = toItemWithEqlPayloads( + { details: { amount__source: 'ct' } }, + orders, + ) + + expect(result).toEqual({ + details: { + amount: { i: { c: 'amount', t: 'orders' }, v: 2, k: 'ct', c: 'ct' }, + }, + }) + }) + it('passes plaintext attributes and null/undefined through untouched', () => { const item = { pk: 'user#1', role: 'admin', a: null, b: undefined } diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index 6ca983bc..3df43c0e 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -265,6 +265,7 @@ export function toItemWithEqlPayloads( attrName: string, attrValue: unknown, isNested: boolean, + prefix = '', ): Record { if (attrValue === null || attrValue === undefined) { return { [attrName]: attrValue } @@ -279,22 +280,33 @@ export function toItemWithEqlPayloads( const encryptedAttrs = Object.keys(encryptConfig.columns) const columnName = attrName.slice(0, -ciphertextAttrSuffix.length) + // A nested attribute's registered column name depends on the authoring + // convention: `encryptedField('example.protected')` (and v3's dotted + // property form) register the full dotted path, whereas + // `encryptedField('amount')` under a `details` group registers just the + // leaf. Prefer the dotted path when the schema knows it, else fall back to + // the leaf — so both conventions rebuild an identifier that matches how the + // column was declared, rather than always guessing the leaf. + const dottedPath = prefix ? `${prefix}.${columnName}` : columnName + const identifierColumn = encryptedAttrs.includes(dottedPath) + ? dottedPath + : columnName + // Handle encrypted payload if ( attrName.endsWith(ciphertextAttrSuffix) && - (encryptedAttrs.includes(columnName) || isNested) + (encryptedAttrs.includes(identifierColumn) || isNested) ) { - const i = { c: columnName, t: encryptConfig.tableName } + const i = { c: identifierColumn, t: encryptConfig.tableName } - // Nested values are not searchable, so we can just return the standard EQL payload. - // Worth noting, that encryptConfig.columns[columnName] will be undefined if isNested is true. + // A JSON document is stored as its ste_vec array and must be rebuilt with + // `k: 'sv'`. Look the config up by the resolved identifier so a nested + // JSON column (registered under a dotted path) is detected too — keyed on + // the leaf it would be missing, and would be rebuilt as a scalar. // A v3 column builds the same `{ cast_as, indexes }` shape as a v2 one, // so this detection needs no version branch. - if ( - !isNested && - encryptConfig.columns[columnName].cast_as === 'json' && - encryptConfig.columns[columnName].indexes.ste_vec - ) { + const columnConfig = encryptConfig.columns[identifierColumn] + if (columnConfig?.cast_as === 'json' && columnConfig.indexes.ste_vec) { return { [columnName]: { i, @@ -320,13 +332,19 @@ export function toItemWithEqlPayloads( } } - // Handle nested objects recursively + // Handle nested objects recursively, carrying the path so a nested column + // can be matched against its registered dotted name. if (typeof attrValue === 'object' && !Array.isArray(attrValue)) { const nestedResult = Object.entries( attrValue as Record, ).reduce( (acc, [key, val]) => { - const processed = processValue(key, val, true) + const processed = processValue( + key, + val, + true, + prefix ? `${prefix}.${attrName}` : attrName, + ) return Object.assign({}, acc, processed) }, {} as Record, diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index 53e4edce..bc9a3a1a 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -51,34 +51,47 @@ Non-encrypted attributes pass through unchanged. On decryption, the `__source` a | Import | `@cipherstash/stack/v3` | `@cipherstash/stack` + `@cipherstash/stack/schema` | | Schema | `encryptedTable` + `types.*` | `encryptedTable` + `encryptedColumn` | | Client | `EncryptionV3({ schemas })` | `Encryption({ schemas })` | -| Nested fields | **Not yet** (deferred) | `encryptedField` | +| Nested fields | Flat dotted path (`"profile.ssn"`) | Nested group + `encryptedField` | There is no data migration between them: DynamoDB has no EQL extension to install and no schema to alter. But the two write **different wire formats**, so a table populated under v2 cannot be read back with a v3 schema, or vice versa. Pick one per table and stay on it. -**Nested objects work differently in each version, and this is the main reason to stay on v2.** +**Nested attributes work in both versions**, with different authoring syntax. -EQL v2's `encryptedField` encrypts *selected leaves in place*. The item keeps its shape and unlisted siblings stay plaintext: +DynamoDB items are natively nested, so this matters here more than on Postgres. Both versions encrypt *selected leaves in place*: the item keeps its shape and unlisted siblings stay plaintext. -```jsonc -// schema: profile: { ssn: encryptedField("profile.ssn") } -{ "profile": { "ssn__source": "", "city": "Sydney" } } +EQL v2 uses a nested group with `encryptedField`: + +```typescript +const users = encryptedTable("users", { + profile: { ssn: encryptedField("profile.ssn") }, +}) ``` -EQL v3 has no `encryptedField` authoring form — a nested group is a compile error, since a v3 column map holds only `types.*` domains. +EQL v3 has no nested-group syntax — a nested object in a v3 column map is a compile error. Declare the column **flat, with a dotted path** instead: + +```typescript +const users = encryptedTable("users", { + "profile.ssn": types.TextEq("profile.ssn"), + "profile.note": types.Text("profile.note"), +}) +``` -The nearest v3 equivalent is `types.Json`, which encrypts the **whole subtree as a single value**: +Both produce the same DynamoDB item, and the v3 form keeps equality queryability on the nested attribute: ```jsonc -// schema: profile: types.Json("profile") -{ "profile__source": [ /* ste_vec entries */ ] } +{ "pk": "u#1", + "profile": { + "ssn__source": "", + "ssn__hmac": "", // key-condition on profile.ssn__hmac + "note__source": "", + "city": "Sydney" // not in schema, stays plaintext + } } ``` -Choose accordingly: +> The dotted string is the *property key* as well as the column name — the model +> is matched by dotted path, so `{ profile: { ssn } }` resolves correctly. -| You want | Use | -|---|---| -| The whole object encrypted as one value | `types.Json` (v3) | -| Some leaves encrypted, siblings plaintext, structure preserved | `encryptedField` (v2 only) | +To encrypt a whole subtree as one value instead of per-leaf, use `types.Json`, which stores it as a single ste_vec attribute (`profile__source`). ## Rolling Encryption Out to Production From 09604fd7fb3a160bc42adca54b573da1ab54cb26 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 20 Jul 2026 20:53:46 +1000 Subject: [PATCH 08/20] fix(stack): correct four data-integrity bugs found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-way review of the branch (architecture, types, tests) found four defects, two of them silent data loss. All are now fixed with regression tests; none were caught by the 80 existing tests because every test table used property === dbName and every test item was well-formed. 1. CRITICAL — a v3 column whose JS property differs from its DB name was silently corrupted. `encryptedAttrs` came from `build().columns`, which for v3 is keyed by DB name, while the encrypted model is keyed by property name. For `emailAddress: types.TextEq('email_address')` the two never matched, so the attribute was never split: stored: { emailAddress: { v:3, i__source:'email_address', c:'…', hm:'…' } } No `__source`, no `__hmac`, and the payload's `i` block rewritten to `i__source` — a violation of the payload-shape contract in AGENTS.md. Equality querying, the adapter's entire purpose, silently did nothing, and round-trip decrypt still passed so nothing surfaced it. Both encrypt paths and the read path now use `resolveEncryptColumnMap` from `encryption/helpers/model-helpers` — the core resolver that exists for exactly this — matching on property names and identifying by DB name. 2. CRITICAL, introduced by this branch — widening the payload predicate from `k === 'ct'` to "has c" meant any nested plaintext object with a `c` key was treated as an envelope: { wrapper: { inner: { c: 'AU', d: 1 } } } -> { wrapper: { inner__source: 'AU' } } `d` silently discarded on write. `c` is an ordinary attribute name (country, currency, count). Now gated on `isStoredEqlPayload`, which requires `v` and `i` — the same keys the FFI itself uses to detect an encrypted value. 3. `deepClone` flattened every non-plain object to `{}`, destroying `Date` before it reached the FFI. That made the entire `types.Timestamp*` / `types.Date*` domain family unusable through this adapter, and circular references threw a stack overflow rather than a handled error. Replaced with `structuredClone`. 4. The read path dropped ANY attribute ending in `__hmac` and treated ANY nested `*__source` as an envelope, both unconditionally. A customer attribute named `signature__hmac` vanished on read; an unregistered nested `*__source` was handed to the FFI as a decrypt target. Both are now scoped to columns the schema actually declares. Also adopts the canonical v3 marker (`buildColumnKeyMap`, per `resolveEqlVersion`) instead of duck-typing `getQueryCapabilities`, and makes `isV3Table` a type predicate. `build()` and the column map are hoisted out of the per-attribute recursion. 89 adapter tests (up from 80), 991 passing overall. --- .../dynamodb/encrypted-dynamodb-v3.test.ts | 60 +++++++ .../__tests__/dynamodb/helpers-v3.test.ts | 74 +++++++++ packages/stack/src/dynamodb/helpers.ts | 151 +++++++++++------- .../operations/bulk-encrypt-models.ts | 10 +- .../src/dynamodb/operations/encrypt-model.ts | 10 +- 5 files changed, 245 insertions(+), 60 deletions(-) diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index b9b7f1ef..6dae8b0d 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -368,6 +368,66 @@ describe('nested attributes with a v3 table', () => { }) }) +describe('a v3 column whose property differs from its DB name', () => { + // Regression: `encryptedAttrs` was derived from `build().columns`, which for + // v3 is keyed by DB name, while the encrypted model is keyed by property + // name. They never matched, so the attribute was never split — no __source, + // no __hmac, and the payload's `i` block was mangled to `i__source`. Decrypt + // still round-tripped, so nothing surfaced it; only a key condition that + // silently matched nothing would have. + const renamed = encryptedTable('users_v3_renamed', { + emailAddress: types.TextEq('email_address'), + }) + + let renamedDynamo: EncryptedDynamoDBInstance + let renamedClient: EncryptionClient + + beforeAll(async () => { + renamedClient = await Encryption({ + schemas: [renamed] as never, + config: { eqlVersion: 3 }, + }) + renamedDynamo = encryptedDynamoDB({ encryptionClient: renamedClient }) + }) + + it('splits the attribute under its property name, with an __hmac', async () => { + const result = await renamedDynamo.encryptModel( + { id: '1', emailAddress: 'a@b.com' }, + renamed, + ) + if (result.failure) throw new Error(result.failure.message) + + expect(Object.keys(result.data).sort()).toEqual([ + 'emailAddress__hmac', + 'emailAddress__source', + 'id', + ]) + // The payload's identifier block must never leak into an attribute. + expect(JSON.stringify(result.data)).not.toContain('i__source') + }) + + it('round-trips, and the query term matches the stored __hmac', async () => { + const original = { id: '2', emailAddress: 'c@d.com' } + + const encrypted = await renamedDynamo.encryptModel(original, renamed) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await renamedDynamo.decryptModel(encrypted.data, renamed) + if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data).toEqual(original) + + const term = await renamedClient.encryptQuery('c@d.com', { + table: renamed as never, + column: renamed.emailAddress as never, + }) + if (term.failure) throw new Error(term.failure.message) + + expect((term.data as { hm: string }).hm).toBe( + (encrypted.data as Record).emailAddress__hmac, + ) + }) +}) + describe('the __hmac key-condition path with a v3 table', () => { it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { const email = 'heidi@example.com' diff --git a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts index 81f5fb57..35d2bc90 100644 --- a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts @@ -15,6 +15,7 @@ */ import { describe, expect, it } from 'vitest' import { + deepClone, isV3Table, toEncryptedDynamoItem, toItemWithEqlPayloads, @@ -205,3 +206,76 @@ describe('toItemWithEqlPayloads for a v3 table', () => { }) }) }) + +describe('regressions found in review', () => { + it('does not mistake a nested plaintext object with a `c` key for a payload', () => { + // `c` is an ordinary attribute name (country, currency, count). Treating + // it as a ciphertext rewrote the object to `__source` and DISCARDED + // every sibling key — silent data loss on a PutCommand. + const item = { shipping: { address: { c: 'AU', street: '1 Main St' } } } + + expect(toEncryptedDynamoItem(item, encryptedAttrs)).toEqual(item) + }) + + it('does not drop a plaintext attribute that merely ends in __hmac', () => { + const item = { pk: 'u#1', signature__hmac: 'an app-level hmac' } + + expect(toItemWithEqlPayloads(item, users)).toEqual(item) + }) + + it('does not treat an unregistered nested __source attribute as an envelope', () => { + const item = { grp: { unrelated__source: 'plaintext' } } + + expect(toItemWithEqlPayloads(item, users)).toEqual(item) + }) + + it('identifies a column by its DB name when it differs from the property', () => { + // `emailAddress: types.TextEq('email_address')` — matching must happen on + // the property name, identification on the DB name. + const t = encryptedTable('t', { + emailAddress: types.TextEq('email_address'), + }) + + const stored = toEncryptedDynamoItem( + { + emailAddress: { + v: 3, + i: { t: 't', c: 'email_address' }, + c: 'CT', + hm: 'H', + }, + }, + Object.keys(t.buildColumnKeyMap()), + ) + expect(stored).toEqual({ + emailAddress__source: 'CT', + emailAddress__hmac: 'H', + }) + + expect(toItemWithEqlPayloads(stored, t)).toEqual({ + emailAddress: { i: { c: 'email_address', t: 't' }, v: 3, c: 'CT' }, + }) + }) +}) + +describe('deepClone preserves structured values', () => { + it('preserves Date instances — the whole Timestamp domain family depends on it', () => { + const at = new Date('2020-01-02T03:04:05.000Z') + + expect(deepClone(at)).toBeInstanceOf(Date) + expect(deepClone({ at }).at.getTime()).toBe(at.getTime()) + }) + + it('preserves Map, Set and typed arrays', () => { + expect(deepClone(new Map([['a', 1]])).get('a')).toBe(1) + expect(deepClone(new Set([1, 2])).has(2)).toBe(true) + expect(deepClone(new Uint8Array([1, 2]))).toBeInstanceOf(Uint8Array) + }) + + it('does not blow the stack on a circular reference', () => { + const o: Record = { a: 1 } + o.self = o + + expect(() => deepClone(o)).not.toThrow() + }) +}) diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index 3df43c0e..db917714 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -1,5 +1,7 @@ import type { ProtectErrorCode } from '@cipherstash/protect-ffi' import { ProtectError as FfiProtectError } from '@cipherstash/protect-ffi' +import { resolveEncryptColumnMap } from '@/encryption/helpers/model-helpers' +import type { AnyV3Table } from '@/eql/v3' import type { EncryptedValue } from '@/types' import { logger } from '@/utils/logger' import type { AnyEncryptedTable, EncryptedDynamoDBError } from './types' @@ -11,25 +13,15 @@ export const searchTermAttrSuffix = '__hmac' * Which EQL wire version to synthesize when rebuilding an envelope from the * stored `__source` attribute. * - * v2 columns are `EncryptedColumn` instances from `@/schema`; v3 columns are - * the concrete-domain classes from `@/eql/v3`, which are the only ones carrying - * `getQueryCapabilities`. That method is the discriminator used elsewhere in - * the codebase (see `encryption/helpers/infer-index-type.ts`), so reuse it - * rather than inventing a second signal. - * - * A v2 table's builders may nest (`encryptedField` groups columns under an - * object), so walk one level rather than assuming a flat map. v3 has no nested - * columns at all. + * `buildColumnKeyMap` is the canonical v3 marker in this codebase — see + * `resolveEqlVersion` (`encryption/index.ts`) and `types.ts`, which document it + * as *the* signal. Only v3 tables define it. */ -export function isV3Table(table: AnyEncryptedTable): boolean { - return Object.values(table.columnBuilders).some(isV3Column) -} - -function isV3Column(builder: unknown): boolean { - if (builder === null || typeof builder !== 'object') return false - if ('getQueryCapabilities' in builder) return true - // A v2 nested group: `{ ssn: encryptedField(...), address: { ... } }`. - return Object.values(builder as Record).some(isV3Column) +export function isV3Table(table: AnyEncryptedTable): table is AnyV3Table { + return ( + 'buildColumnKeyMap' in table && + typeof table.buildColumnKeyMap === 'function' + ) } export class EncryptedDynamoDBErrorImpl @@ -138,23 +130,28 @@ export function throwPreservingCode(failure: { throw error } +/** + * Clone caller input before it reaches the FFI, so encryption never mutates a + * caller's object. + * + * `structuredClone` rather than a hand-rolled `Object.entries` reduce: the + * reduce flattened every non-plain object to `{}`, which silently destroyed + * `Date` values — making the whole `types.Timestamp*` / `types.Date*` domain + * family unusable through this adapter — and blew the stack on a circular + * reference. `structuredClone` handles Date, Map, Set, TypedArray and cycles + * natively. Values it cannot structurally clone fall back to the original + * reference rather than throwing. + */ export function deepClone(obj: T): T { if (obj === null || typeof obj !== 'object') { return obj } - if (Array.isArray(obj)) { - return obj.map((item) => deepClone(item)) as unknown as T + try { + return structuredClone(obj) + } catch { + return obj } - - return Object.entries(obj as Record).reduce( - (acc, [key, value]) => ({ - // biome-ignore lint/performance/noAccumulatingSpread: TODO later - ...acc, - [key]: deepClone(value), - }), - {} as T, - ) } /** @@ -168,6 +165,10 @@ export function deepClone(obj: T): T { * keys, so state them structurally. */ type StoredEqlPayload = { + /** Wire version. Present on every payload in both versions. */ + v?: unknown + /** Identifier `{ t, c }`. Present on every payload in both versions. */ + i?: unknown /** Absent on v3 scalars; `'ct'` on v2 scalars; `'sv'` on JSON in both. */ k?: 'ct' | 'sv' /** Scalar ciphertext. */ @@ -178,6 +179,23 @@ type StoredEqlPayload = { sv?: unknown } +/** + * Is this value an EQL payload, as opposed to a plaintext object that merely + * looks like one? + * + * `v` and `i` are the discriminators, deliberately: they are the same keys the + * FFI itself uses to decide whether a value is encrypted, and every payload in + * both wire versions carries both. Testing for a ciphertext alone is not + * enough — `c` is an ordinary attribute name (country, currency, count), and + * treating `{ c: 'AU', d: 1 }` as a payload silently rewrites it to + * `__source` and DISCARDS every sibling key. + */ +function isStoredEqlPayload(value: unknown): value is StoredEqlPayload { + if (value === null || typeof value !== 'object') return false + if (!('v' in value) || !('i' in value)) return false + return 'c' in value || 'sv' in value +} + export function toEncryptedDynamoItem( encrypted: Record, encryptedAttrs: string[], @@ -191,14 +209,14 @@ export function toEncryptedDynamoItem( return { [attrName]: attrValue } } - // Handle encrypted payload + // Handle encrypted payload. Both arms require the value to actually BE a + // payload — a registered attribute name is not sufficient on its own, and + // for a nested value the name tells us nothing at all. if ( - encryptedAttrs.includes(attrName) || - (isNested && - typeof attrValue === 'object' && - 'c' in (attrValue as object)) + (encryptedAttrs.includes(attrName) || isNested) && + isStoredEqlPayload(attrValue) ) { - const encryptPayload = attrValue as StoredEqlPayload + const encryptPayload = attrValue // A JSON document, in either wire version, keeps its `k: 'sv'` tag. Its // index terms live *inside* the `sv` entries, so the whole array is @@ -260,6 +278,25 @@ export function toItemWithEqlPayloads( encryptionSchema: AnyEncryptedTable, ): Record { const v = isV3Table(encryptionSchema) ? 3 : 2 + // Hoisted: `build()` is not memoized, and this used to run once per attribute + // per nesting level per item. + const encryptConfig = encryptionSchema.build() + // `columnPaths` are the keys a model's fields are MATCHED on (JS property + // names); `toColumnName` maps a match to the name the FFI config is keyed by. + // For v3 those differ whenever a column is declared as + // `emailAddress: types.TextEq('email_address')` — matching on the config keys + // instead would miss the attribute entirely, drop its `__hmac`, and recurse + // into the payload. Reuses the core resolver rather than re-deriving it. + const { columnPaths, toColumnName } = + resolveEncryptColumnMap(encryptionSchema) + + /** Resolve an attribute back to the column path it was written from. */ + function matchColumn(leaf: string, prefix: string): string | undefined { + const dotted = prefix ? `${prefix}.${leaf}` : leaf + if (columnPaths.includes(dotted)) return dotted + if (columnPaths.includes(leaf)) return leaf + return undefined + } function processValue( attrName: string, @@ -271,33 +308,31 @@ export function toItemWithEqlPayloads( return { [attrName]: attrValue } } - // Skip HMAC fields + // Drop the search term — but only when it belongs to a column we know + // about. An unrelated customer attribute that merely ends in `__hmac` + // (an app-level signature, say) must survive the read untouched. if (attrName.endsWith(searchTermAttrSuffix)) { - return {} + const term = attrName.slice(0, -searchTermAttrSuffix.length) + if (matchColumn(term, prefix)) return {} + return { [attrName]: attrValue } } - const encryptConfig = encryptionSchema.build() - const encryptedAttrs = Object.keys(encryptConfig.columns) const columnName = attrName.slice(0, -ciphertextAttrSuffix.length) - // A nested attribute's registered column name depends on the authoring - // convention: `encryptedField('example.protected')` (and v3's dotted - // property form) register the full dotted path, whereas - // `encryptedField('amount')` under a `details` group registers just the - // leaf. Prefer the dotted path when the schema knows it, else fall back to - // the leaf — so both conventions rebuild an identifier that matches how the - // column was declared, rather than always guessing the leaf. - const dottedPath = prefix ? `${prefix}.${columnName}` : columnName - const identifierColumn = encryptedAttrs.includes(dottedPath) - ? dottedPath - : columnName - - // Handle encrypted payload - if ( - attrName.endsWith(ciphertextAttrSuffix) && - (encryptedAttrs.includes(identifierColumn) || isNested) - ) { - const i = { c: identifierColumn, t: encryptConfig.tableName } + // Resolve the attribute back to a declared column. `matchColumn` prefers + // the dotted path (`encryptedField('example.protected')`, and v3's dotted + // property form) and falls back to the bare leaf (`encryptedField('amount')` + // under a `details` group), so both authoring conventions resolve. + const matched = matchColumn(columnName, prefix) + + // Handle encrypted payload. An unmatched attribute is NOT an envelope, even + // when nested — previously `|| isNested` made every nested `*__source` a + // decrypt target, so an unrelated customer attribute was handed to the FFI. + if (attrName.endsWith(ciphertextAttrSuffix) && matched) { + // Match on the property name, but identify by the DB column name — they + // differ whenever a v3 column is declared + // `emailAddress: types.TextEq('email_address')`. + const i = { c: toColumnName(matched), t: encryptConfig.tableName } // A JSON document is stored as its ste_vec array and must be rebuilt with // `k: 'sv'`. Look the config up by the resolved identifier so a nested @@ -305,7 +340,7 @@ export function toItemWithEqlPayloads( // the leaf it would be missing, and would be rebuilt as a scalar. // A v3 column builds the same `{ cast_as, indexes }` shape as a v2 one, // so this detection needs no version branch. - const columnConfig = encryptConfig.columns[identifierColumn] + const columnConfig = encryptConfig.columns[toColumnName(matched)] if (columnConfig?.cast_as === 'json' && columnConfig.indexes.ste_vec) { return { [columnName]: { diff --git a/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts b/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts index bf976569..ff23b921 100644 --- a/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts +++ b/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts @@ -1,4 +1,5 @@ import { type Result, withResult } from '@byteslice/result' +import { resolveEncryptColumnMap } from '@/encryption/helpers/model-helpers' import { logger } from '@/utils/logger' import { deepClone, @@ -53,7 +54,14 @@ export class BulkEncryptModelsOperation< } const data = encryptResult.data.map((item) => deepClone(item)) - const encryptedAttrs = Object.keys(this.table.build().columns) + // Property names, NOT `build().columns` keys: for v3 those are the DB + // column names, which differ whenever a column is declared + // `emailAddress: types.TextEq('email_address')`. The encrypted model + // coming back from the client is keyed by property name, so matching on + // config keys silently missed the attribute entirely. + const { columnPaths: encryptedAttrs } = resolveEncryptColumnMap( + this.table, + ) return data.map( (encrypted) => toEncryptedDynamoItem(encrypted, encryptedAttrs) as T, diff --git a/packages/stack/src/dynamodb/operations/encrypt-model.ts b/packages/stack/src/dynamodb/operations/encrypt-model.ts index efe36e68..469988f4 100644 --- a/packages/stack/src/dynamodb/operations/encrypt-model.ts +++ b/packages/stack/src/dynamodb/operations/encrypt-model.ts @@ -1,4 +1,5 @@ import { type Result, withResult } from '@byteslice/result' +import { resolveEncryptColumnMap } from '@/encryption/helpers/model-helpers' import { logger } from '@/utils/logger' import { deepClone, @@ -50,7 +51,14 @@ export class EncryptModelOperation< } const data = deepClone(encryptResult.data) - const encryptedAttrs = Object.keys(this.table.build().columns) + // Property names, NOT `build().columns` keys: for v3 those are the DB + // column names, which differ whenever a column is declared + // `emailAddress: types.TextEq('email_address')`. The encrypted model + // coming back from the client is keyed by property name, so matching on + // config keys silently missed the attribute entirely. + const { columnPaths: encryptedAttrs } = resolveEncryptColumnMap( + this.table, + ) return toEncryptedDynamoItem(data, encryptedAttrs) as T }, From c32720d79147080d4501dd36705d4e3d42a8d8fe Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 21 Jul 2026 08:44:49 +1000 Subject: [PATCH 09/20] feat(stack): strongly type the DynamoDB v3 surface + property tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the type-surface and test-level gaps a three-way review flagged after the runtime fixes landed. Types (src/dynamodb/types.ts, index.ts) — zero runtime change: - encryptModel / bulkEncryptModels / decryptModel / bulkDecryptModels are now overloaded per wire version. The v3 arm derives the model from the table: input via `V3ModelInput` (schema fields pinned to their domain plaintext, pk/sk/GSI keys passing through), output via a new `EncryptedAttributes` mapped type that expresses the storage split — `__source` plus an OPTIONAL `__hmac`, present only for domains that mint one (`HasSearchTerm` mirrors `indexesForCapabilities`: the *Eq family plus TextOrd/TextOrdOre/TextSearch). `DecryptedAttributes` is the inverse. The v2 arm is byte-identical to before, so existing callers are untouched. This replaces `encryptModel(item, table): EncryptModelOperation`, which both left `T` unrelated to the table (`{ emial: 42 }` compiled) and lied about the return (`result.data.email` type-checked but was undefined at runtime; `result.data.email__source` did not type-check). - `DynamoDBEncryptionClient.decryptModel`/`bulkDecryptModels` take `table: never` (required) not `table?: never`. The optional form collapsed to `undefined`, which the required-table `TypedEncryptionClient` did not satisfy — so the type's own claim that both clients fit without a cast was false. - Export `AnyEncryptedTable`, `DynamoDBEncryptionClient`, `EncryptedAttributes`, `DecryptedAttributes`, `AuditConfig` — all appeared in public signatures but were un-nameable by consumers. Suffix literals for the mapped types are sourced from the helpers.ts constants via a type-only import, so the type and the runtime cannot drift. Tests: - All 26 type errors in the dynamodb test files are gone (they were the old return type being unsound). No model type was widened with an index signature to hide a defect: the v2 test names the real wire shape as `StoredUser`, and one genuine defect surfaced and was tightened — a `Record` model against a `types.Json` column, now `Record`. - client-compat.test-d.ts (16 assertions) locks the claim that had none: both client shapes assign with no cast, all four methods take a v3 and a v2 table, .audit() chains, awaiting yields a discriminated Result, a bare object is rejected. This is the M1-class gap vitest.config.ts warns about. - properties.test.ts (14 pure fast-check properties, ~200ms) pins each bug the review found: passthrough with hostile keys (c/hm/v/i/__hmac) at 1000 runs, property-vs-DB-name, deepClone/Date, version-follows-table, no metadata leak. Verified non-vacuous by mutation: reintroducing the four bugs fails exactly 6 properties. A capped (numRuns: 5) live round-trip property covers the mapping composed with real encryption. test:types 80 (was 64); dynamodb suite 104 (was 89); 1006 passing overall. Two documented type limitations, both runtime-correct: the v2 overload still returns the input model (v2 columns carry no capability in the type), and a nested dotted-path v3 column (`'profile.ssn'`) splits correctly at runtime but the model type is silent about it. --- .changeset/dynamodb-eql-v3.md | 10 + .../dynamodb/client-compat.test-d.ts | 225 ++++++++ .../dynamodb/encrypted-dynamodb-v3.test.ts | 139 +++-- .../dynamodb/encrypted-dynamodb.test.ts | 59 +- .../stack/__tests__/dynamodb/helpers.test.ts | 2 +- .../__tests__/dynamodb/properties.test.ts | 525 ++++++++++++++++++ packages/stack/src/dynamodb/index.ts | 15 +- packages/stack/src/dynamodb/types.ts | 172 +++++- skills/stash-dynamodb/SKILL.md | 27 +- 9 files changed, 1091 insertions(+), 83 deletions(-) create mode 100644 packages/stack/__tests__/dynamodb/client-compat.test-d.ts create mode 100644 packages/stack/__tests__/dynamodb/properties.test.ts diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md index 67d9a7c2..ee5ba240 100644 --- a/.changeset/dynamodb-eql-v3.md +++ b/.changeset/dynamodb-eql-v3.md @@ -38,3 +38,13 @@ Notes on capability: The DynamoDB adapter also gains its first test coverage — 74 tests across the v2 and v3 paths, where it previously had none. + +The v3 overloads are strongly typed. `encryptModel` / `bulkEncryptModels` check +the input model against the table's column domains, and return the DynamoDB +attribute map that is actually written — the new exported `EncryptedAttributes` +type, where a declared column `email` becomes `email__source` (plus +`email__hmac` for the equality domains that mint one) rather than surviving as +`email`. `decryptModel` / `bulkDecryptModels` invert it via `DecryptedAttributes`. +`AnyEncryptedTable`, `DynamoDBEncryptionClient` and `AuditConfig` are now +exported from `@cipherstash/stack/dynamodb` so these signatures can be named. +The EQL v2 overloads are unchanged. diff --git a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts new file mode 100644 index 00000000..7f6b54a2 --- /dev/null +++ b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts @@ -0,0 +1,225 @@ +/** + * Type-level contract for `encryptedDynamoDB` (#657). + * + * The runtime `*.test.ts` suites are NOT typechecked (see the comment in + * `vitest.config.ts`), which is exactly how an M1-class regression slips + * through: `encryptedDynamoDB` rejecting the `TypedEncryptionClient` that + * `EncryptionV3` returns compiles nowhere and fails nothing, because the live + * suite that would have caught it is only ever *executed*. + * + * So the claims the adapter's own docs make — both client shapes accepted with + * no cast, both wire versions accepted on all four methods, `.audit()` chains, + * awaiting yields a discriminated Result — are locked here, where `tsc` runs. + */ +import { describe, expectTypeOf, it } from 'vitest' +import type { EncryptedDynamoDBInstance } from '@/dynamodb' +import { encryptedDynamoDB } from '@/dynamodb' +import type { EncryptionClient } from '@/encryption' +import type { EncryptionV3 } from '@/encryption/v3' +import { encryptedTable as encryptedTableV3, types } from '@/eql/v3' +import { encryptedColumn, encryptedTable as encryptedTableV2 } from '@/schema' + +const usersV3 = encryptedTableV3('users_v3', { + email: types.TextEq('email'), // equality → __source + __hmac + name: types.Text('name'), // storage only → __source + age: types.IntegerOrd('age'), // ordering, no HMAC → __source + meta: types.Json('meta'), // ste_vec array → __source +}) + +const usersV2 = encryptedTableV2('users_v2', { + email: encryptedColumn('email').equality(), +}) + +type V3Model = { pk: string; email?: string; age?: number; role?: string } + +// The two client shapes. `EncryptionV3` returns a `TypedEncryptionClient` +// parameterised by its own schema tuple; `Encryption` returns the nominal +// `EncryptionClient`. Both must be accepted by the factory WITHOUT a cast. +declare const typedClient: Awaited< + ReturnType> +> +declare const nominalClient: EncryptionClient + +describe('encryptedDynamoDB accepts both client shapes without a cast', () => { + it('accepts the typed client from EncryptionV3', () => { + expectTypeOf(encryptedDynamoDB).toBeCallableWith({ + encryptionClient: typedClient, + }) + expectTypeOf( + encryptedDynamoDB({ encryptionClient: typedClient }), + ).toEqualTypeOf() + }) + + it('accepts the nominal EncryptionClient', () => { + expectTypeOf(encryptedDynamoDB).toBeCallableWith({ + encryptionClient: nominalClient, + }) + expectTypeOf( + encryptedDynamoDB({ encryptionClient: nominalClient }), + ).toEqualTypeOf() + }) + + it('rejects a bare object that is not a client', () => { + encryptedDynamoDB({ + // @ts-expect-error - not an encryption client + encryptionClient: { nope: true }, + }) + }) +}) + +const dynamo = encryptedDynamoDB({ encryptionClient: typedClient }) + +describe('all four methods accept both a v3 and a v2 table', () => { + it('encryptModel', () => { + expectTypeOf(dynamo.encryptModel).toBeCallableWith( + { pk: 'a', email: 'a@b.com' }, + usersV3, + ) + expectTypeOf(dynamo.encryptModel).toBeCallableWith( + { pk: 'a', email: 'a@b.com' }, + usersV2, + ) + }) + + it('bulkEncryptModels', () => { + expectTypeOf(dynamo.bulkEncryptModels).toBeCallableWith( + [{ pk: 'a', email: 'a@b.com' }], + usersV3, + ) + expectTypeOf(dynamo.bulkEncryptModels).toBeCallableWith( + [{ pk: 'a', email: 'a@b.com' }], + usersV2, + ) + }) + + it('decryptModel', () => { + expectTypeOf(dynamo.decryptModel).toBeCallableWith( + { pk: 'a', email__source: 'ct' }, + usersV3, + ) + expectTypeOf(dynamo.decryptModel).toBeCallableWith( + { pk: 'a', email__source: 'ct' }, + usersV2, + ) + }) + + it('bulkDecryptModels', () => { + expectTypeOf(dynamo.bulkDecryptModels).toBeCallableWith( + [{ pk: 'a', email__source: 'ct' }], + usersV3, + ) + expectTypeOf(dynamo.bulkDecryptModels).toBeCallableWith( + [{ pk: 'a', email__source: 'ct' }], + usersV2, + ) + }) + + it('rejects a bare object as a table', () => { + dynamo.encryptModel( + { pk: 'a' }, + // @ts-expect-error - a plain object is not an encrypted table + { email: 'not-a-table' }, + ) + }) +}) + +describe('the v3 overload types the DynamoDB storage split', () => { + it('replaces a declared column with __source, and mints __hmac only for equality', async () => { + const result = await dynamo.encryptModel( + { pk: 'a', email: 'a@b.com', age: 3, role: 'admin' }, + usersV3, + ) + if (result.failure) return + + // Equality domain: ciphertext + the queryable HMAC term. + expectTypeOf(result.data.email__source).toEqualTypeOf() + expectTypeOf(result.data.email__hmac).toEqualTypeOf() + // Ordering domain: ciphertext only — `op` has no DynamoDB query surface. + expectTypeOf(result.data.age__source).toEqualTypeOf() + expectTypeOf(result.data).not.toHaveProperty('age__hmac') + // The plaintext key is GONE — typing it as present is the defect this fixes. + expectTypeOf(result.data).not.toHaveProperty('email') + // Non-schema keys pass through untouched (pk / sk / GSI attributes). + expectTypeOf(result.data.pk).toEqualTypeOf() + expectTypeOf(result.data.role).toEqualTypeOf() + }) + + it('types a JSON document as its stored ste_vec array', async () => { + const result = await dynamo.encryptModel( + { pk: 'a', meta: { a: 1 } }, + usersV3, + ) + if (result.failure) return + + expectTypeOf(result.data.meta__source).toEqualTypeOf() + expectTypeOf(result.data).not.toHaveProperty('meta__hmac') + }) + + it('rejects a field whose type does not match its column domain', () => { + // @ts-expect-error - `email` is a text domain, not a number + dynamo.encryptModel({ pk: 'a', email: 42 }, usersV3) + }) + + it('folds the stored attributes back to plaintext on decrypt', async () => { + const result = await dynamo.decryptModel( + { pk: 'a', email__source: 'ct', email__hmac: 'hm', role: 'admin' }, + usersV3, + ) + if (result.failure) return + + expectTypeOf(result.data.email).toEqualTypeOf() + // The query term is not data — it does not survive the read. + expectTypeOf(result.data).not.toHaveProperty('email__hmac') + expectTypeOf(result.data).not.toHaveProperty('email__source') + expectTypeOf(result.data.pk).toEqualTypeOf() + expectTypeOf(result.data.role).toEqualTypeOf() + }) + + it('round-trips a declared model shape', async () => { + const model: V3Model = { pk: 'a', email: 'a@b.com', role: 'admin' } + const encrypted = await dynamo.encryptModel(model, usersV3) + if (encrypted.failure) return + + expectTypeOf(dynamo.decryptModel).toBeCallableWith(encrypted.data, usersV3) + }) +}) + +describe('the v2 overload still returns the input model', () => { + it('keeps an existing v2 caller compiling unchanged', async () => { + const result = await dynamo.encryptModel<{ pk: string; email?: string }>( + { pk: 'a', email: 'a@b.com' }, + usersV2, + ) + if (result.failure) return + + expectTypeOf(result.data).toEqualTypeOf<{ pk: string; email?: string }>() + }) +}) + +describe('operations chain and resolve', () => { + it('.audit() returns the operation so it stays chainable', () => { + const op = dynamo.encryptModel({ pk: 'a', email: 'a@b.com' }, usersV3) + expectTypeOf(op.audit({ metadata: { sub: 'u1' } })).toEqualTypeOf< + typeof op + >() + }) + + it('awaiting yields a discriminated Result', async () => { + const result = await dynamo.encryptModel( + { pk: 'a', email: 'a@b.com' }, + usersV3, + ) + + if (result.failure) { + expectTypeOf(result.failure.message).toEqualTypeOf() + // The success arm is not reachable in this branch — `data` is not even a + // property of the failure member, which is the discrimination working. + expectTypeOf(result).not.toHaveProperty('data') + return + } + + // ...and the failure arm is unreachable in this one. + expectTypeOf(result.failure).toEqualTypeOf() + expectTypeOf(result.data.email__source).toEqualTypeOf() + }) +}) diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index 6dae8b0d..9d3d7e10 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -15,12 +15,14 @@ * Requires CipherStash credentials (`CS_*`), like the rest of this suite. */ import 'dotenv/config' +import fc from 'fast-check' import { beforeAll, describe, expect, it } from 'vitest' import { encryptedDynamoDB } from '@/dynamodb' import { toItemWithEqlPayloads } from '@/dynamodb/helpers' import type { EncryptedDynamoDBInstance } from '@/dynamodb/types' import type { EncryptionClient } from '@/encryption' import { EncryptionV3 } from '@/encryption/v3' +import type { JsonValue } from '@/eql/v3' import { encryptedTable, types } from '@/eql/v3' import { Encryption } from '@/index' @@ -38,7 +40,10 @@ type User = { name?: string age?: number bio?: string - meta?: Record + // `Record` would not compile against a `types.Json` column: + // `unknown` is not a `JsonValue`. Naming the real element type is the point of + // the v3 overload — it catches a model that claims to hold non-JSON. + meta?: Record role?: string } @@ -61,7 +66,7 @@ beforeAll(async () => { describe('encryptModel with a v3 table', () => { it('splits an equality domain into __source and __hmac', async () => { - const result = await typedDynamo.encryptModel( + const result = await typedDynamo.encryptModel( { pk: 'user#1', email: 'alice@example.com', role: 'admin' }, users, ) @@ -80,7 +85,7 @@ describe('encryptModel with a v3 table', () => { }) it('regression: a v3 scalar is split, not written out as a raw map', async () => { - const result = await typedDynamo.encryptModel( + const result = await typedDynamo.encryptModel( { pk: 'user#2', name: 'Bob' }, users, ) @@ -96,7 +101,7 @@ describe('encryptModel with a v3 table', () => { }) it('gives an ordering domain a __source but no __hmac', async () => { - const result = await typedDynamo.encryptModel( + const result = await typedDynamo.encryptModel( { pk: 'user#3', age: 42 }, users, ) @@ -110,7 +115,7 @@ describe('encryptModel with a v3 table', () => { }) it('keeps only the equality term of a TextSearch domain', async () => { - const result = await typedDynamo.encryptModel( + const result = await typedDynamo.encryptModel( { pk: 'user#4', bio: 'a long biography' }, users, ) @@ -126,7 +131,7 @@ describe('encryptModel with a v3 table', () => { }) it('stores a Json domain as a ste_vec array', async () => { - const result = await typedDynamo.encryptModel( + const result = await typedDynamo.encryptModel( { pk: 'user#5', meta: { a: 1, b: { c: 'deep' } } }, users, ) @@ -134,7 +139,7 @@ describe('encryptModel with a v3 table', () => { if (result.failure) throw new Error(result.failure.message) expect(Array.isArray(result.data.meta__source)).toBe(true) - expect((result.data.meta__source as unknown[]).length).toBeGreaterThan(0) + expect(result.data.meta__source.length).toBeGreaterThan(0) }) }) @@ -150,13 +155,10 @@ describe('round trips with a v3 table', () => { role: 'admin', } - const encrypted = await typedDynamo.encryptModel(original, users) + const encrypted = await typedDynamo.encryptModel(original, users) if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await typedDynamo.decryptModel( - encrypted.data, - users, - ) + const decrypted = await typedDynamo.decryptModel(encrypted.data, users) if (decrypted.failure) throw new Error(decrypted.failure.message) expect(decrypted.data).toEqual(original) @@ -170,13 +172,10 @@ describe('round trips with a v3 table', () => { meta: { z: 1 }, } - const encrypted = await nominalDynamo.encryptModel(original, users) + const encrypted = await nominalDynamo.encryptModel(original, users) if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo.decryptModel( - encrypted.data, - users, - ) + const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) if (decrypted.failure) throw new Error(decrypted.failure.message) expect(decrypted.data).toEqual(original) @@ -185,17 +184,59 @@ describe('round trips with a v3 table', () => { it('produces items either client can decrypt — the wire format is the same', async () => { const original: User = { pk: 'user#8', email: 'grace@example.com' } - const encrypted = await typedDynamo.encryptModel(original, users) + const encrypted = await typedDynamo.encryptModel(original, users) if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo.decryptModel( - encrypted.data, - users, - ) + const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) if (decrypted.failure) throw new Error(decrypted.failure.message) expect(decrypted.data).toEqual(original) }) + + /** + * The one LIVE property. Its pure siblings (`properties.test.ts`) cover the + * attribute mapping in both directions with a fake ciphertext; this closes the + * loop over the real thing — every run is a ZeroKMS encrypt + decrypt, so it + * is capped hard at 15 runs. It is the only check that the split/rebuild + * survives contact with actual ciphertext across all five domains at once. + */ + it('property: round-trips arbitrary multi-domain items through ZeroKMS', async () => { + const jsonLeaf = fc.oneof(fc.string(), fc.integer(), fc.boolean()) + const userArb = fc.record({ + pk: fc.string({ minLength: 1 }), + email: fc.string(), + name: fc.string(), + age: fc.integer(), + bio: fc.string(), + meta: fc.dictionary( + fc.string({ minLength: 1 }), + fc.oneof( + jsonLeaf, + fc.dictionary(fc.string({ minLength: 1 }), jsonLeaf, { maxKeys: 2 }), + ), + { minKeys: 1, maxKeys: 3 }, + ), + role: fc.string(), + }) + + await fc.assert( + fc.asyncProperty(userArb, async (original) => { + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await typedDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }), + // Deliberately small: each run is a real ZeroKMS round trip. The pure + // properties in `properties.test.ts` cover the attribute mapping + // exhaustively and for free; this one only needs to establish that the + // mapping composed with real encryption is identity-preserving over + // arbitrary items, which a handful of runs does. + { numRuns: 5 }, + ) + }, 120000) }) describe('bulk operations with a v3 table', () => { @@ -205,7 +246,7 @@ describe('bulk operations with a v3 table', () => { { pk: 'user#10', email: 'b@example.com', age: 3 }, ] - const encrypted = await typedDynamo.bulkEncryptModels(items, users) + const encrypted = await typedDynamo.bulkEncryptModels(items, users) if (encrypted.failure) throw new Error(encrypted.failure.message) expect(encrypted.data).toHaveLength(2) @@ -214,10 +255,7 @@ describe('bulk operations with a v3 table', () => { expect(item).toHaveProperty('email__hmac') } - const decrypted = await typedDynamo.bulkDecryptModels( - encrypted.data, - users, - ) + const decrypted = await typedDynamo.bulkDecryptModels(encrypted.data, users) if (decrypted.failure) throw new Error(decrypted.failure.message) expect(decrypted.data).toEqual(items) @@ -229,10 +267,10 @@ describe('bulk operations with a v3 table', () => { { pk: 'user#12', name: 'D' }, ] - const encrypted = await nominalDynamo.bulkEncryptModels(items, users) + const encrypted = await nominalDynamo.bulkEncryptModels(items, users) if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo.bulkDecryptModels( + const decrypted = await nominalDynamo.bulkDecryptModels( encrypted.data, users, ) @@ -271,7 +309,7 @@ describe('nested attributes with a v3 table', () => { }) it('splits nested attributes in place, leaving plaintext siblings alone', async () => { - const result = await nestedDynamo.encryptModel( + const result = await nestedDynamo.encryptModel( { pk: 'u#1', profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, @@ -297,23 +335,17 @@ describe('nested attributes with a v3 table', () => { profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, } - const encrypted = await nestedDynamo.encryptModel( - original, - nested, - ) + const encrypted = await nestedDynamo.encryptModel(original, nested) if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nestedDynamo.decryptModel( - encrypted.data, - nested, - ) + const decrypted = await nestedDynamo.decryptModel(encrypted.data, nested) if (decrypted.failure) throw new Error(decrypted.failure.message) expect(decrypted.data).toEqual(original) }) it('rebuilds the nested identifier from its registered dotted name', async () => { - const encrypted = await nestedDynamo.encryptModel( + const encrypted = await nestedDynamo.encryptModel( { pk: 'u#3', profile: { ssn: '123-45-6789' } }, nested, ) @@ -329,7 +361,7 @@ describe('nested attributes with a v3 table', () => { it('makes a nested equality attribute queryable by __hmac', async () => { const ssn = '999-88-7777' - const stored = await nestedDynamo.encryptModel( + const stored = await nestedDynamo.encryptModel( { pk: 'u#4', profile: { ssn } }, nested, ) @@ -352,13 +384,10 @@ describe('nested attributes with a v3 table', () => { { pk: 'u#6', profile: { note: 'b' } }, ] - const encrypted = await nestedDynamo.bulkEncryptModels( - items, - nested, - ) + const encrypted = await nestedDynamo.bulkEncryptModels(items, nested) if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nestedDynamo.bulkDecryptModels( + const decrypted = await nestedDynamo.bulkDecryptModels( encrypted.data, nested, ) @@ -432,7 +461,7 @@ describe('the __hmac key-condition path with a v3 table', () => { it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { const email = 'heidi@example.com' - const stored = await typedDynamo.encryptModel( + const stored = await typedDynamo.encryptModel( { pk: 'user#13', email }, users, ) @@ -453,14 +482,8 @@ describe('the __hmac key-condition path with a v3 table', () => { it('is deterministic across separate encryptions of the same value', async () => { const email = 'ivan@example.com' - const first = await typedDynamo.encryptModel( - { pk: 'a', email }, - users, - ) - const second = await typedDynamo.encryptModel( - { pk: 'b', email }, - users, - ) + const first = await typedDynamo.encryptModel({ pk: 'a', email }, users) + const second = await typedDynamo.encryptModel({ pk: 'b', email }, users) if (first.failure) throw new Error(first.failure.message) if (second.failure) throw new Error(second.failure.message) @@ -476,12 +499,12 @@ describe('audit metadata with a v3 table', () => { const item: User = { pk: 'user#14', email: 'judy@example.com' } const encrypted = await nominalDynamo - .encryptModel(item, users) + .encryptModel(item, users) .audit({ metadata }) if (encrypted.failure) throw new Error(encrypted.failure.message) const decrypted = await nominalDynamo - .decryptModel(encrypted.data, users) + .decryptModel(encrypted.data, users) .audit({ metadata }) if (decrypted.failure) throw new Error(decrypted.failure.message) @@ -492,7 +515,7 @@ describe('audit metadata with a v3 table', () => { const item: User = { pk: 'user#15', email: 'ken@example.com' } const encrypted = await typedDynamo - .encryptModel(item, users) + .encryptModel(item, users) .audit({ metadata }) if (encrypted.failure) throw new Error(encrypted.failure.message) @@ -500,7 +523,7 @@ describe('audit metadata with a v3 table', () => { // surface. The chain must still resolve correctly — the metadata is simply // not forwarded. Use the nominal client if decrypt audit matters. const decrypted = await typedDynamo - .decryptModel(encrypted.data, users) + .decryptModel(encrypted.data, users) .audit({ metadata }) if (decrypted.failure) throw new Error(decrypted.failure.message) diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts index 2cd304fe..78bdd3bb 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts @@ -24,7 +24,7 @@ import { encryptedColumn, encryptedField, encryptedTable } from '@/schema' const users = encryptedTable('users', { email: encryptedColumn('email').equality(), name: encryptedColumn('name'), - doc: encryptedColumn('doc').dataType('json').searchableJson('users/doc'), + doc: encryptedColumn('doc').dataType('json').searchableJson(), example: { protected: encryptedField('example.protected'), }, @@ -39,6 +39,29 @@ type User = { example?: { protected?: string | null; notProtected?: string } } +/** + * The attribute map `encryptModel` actually writes for the v2 `users` table. + * + * The v2 overload still returns the INPUT model type. Unlike v3, a v2 column + * does not carry its index configuration in the type, so the `__source` / + * `__hmac` split cannot be derived the way `EncryptedAttributes` derives it for + * a v3 table — see the note on `EncryptedDynamoDBInstance.encryptModel`. + * Naming the wire shape here, rather than widening `User` with an index + * signature, keeps these assertions honest about what is actually stored. + */ +type StoredUser = { + pk: string + role?: string + email__source?: string + email__hmac?: string + name__source?: string + doc__source?: unknown[] + example?: { protected__source?: string; notProtected?: string } +} + +/** Read a v2 encrypt result as the attribute map it really is. */ +const storedAttrs = (item: User): StoredUser => item as StoredUser + let client: EncryptionClient let dynamo: EncryptedDynamoDBInstance @@ -70,11 +93,15 @@ describe('encryptModel', () => { ]) expect(result.data.pk).toBe('user#1') expect(result.data.role).toBe('admin') - expect(typeof result.data.email__source).toBe('string') - expect(typeof result.data.email__hmac).toBe('string') + expect(typeof storedAttrs(result.data).email__source).toBe('string') + expect(typeof storedAttrs(result.data).email__hmac).toBe('string') // Neither stored attribute leaks the plaintext. - expect(result.data.email__source).not.toContain('alice@example.com') - expect(result.data.email__hmac).not.toContain('alice@example.com') + expect(storedAttrs(result.data).email__source).not.toContain( + 'alice@example.com', + ) + expect(storedAttrs(result.data).email__hmac).not.toContain( + 'alice@example.com', + ) }) it('gives a non-equality column a __source but no __hmac', async () => { @@ -97,8 +124,10 @@ describe('encryptModel', () => { if (result.failure) throw new Error(result.failure.message) - expect(Array.isArray(result.data.doc__source)).toBe(true) - expect((result.data.doc__source as unknown[]).length).toBeGreaterThan(0) + expect(Array.isArray(storedAttrs(result.data).doc__source)).toBe(true) + expect( + (storedAttrs(result.data).doc__source as unknown[]).length, + ).toBeGreaterThan(0) }) it('encrypts a nested field in place, keeping siblings plaintext', async () => { @@ -258,7 +287,9 @@ describe('the __hmac key-condition path', () => { // This equality is the whole DynamoDB query story: a caller puts // `term.data.hm` into `KeyConditionExpression: "email__hmac = :e"` and it // matches the attribute written at encrypt time. - expect((term.data as { hm: string }).hm).toBe(stored.data.email__hmac) + expect((term.data as { hm: string }).hm).toBe( + storedAttrs(stored.data).email__hmac, + ) }) it('mints a different HMAC for a different plaintext', async () => { @@ -275,7 +306,9 @@ describe('the __hmac key-condition path', () => { ) if (stored.failure) throw new Error(stored.failure.message) - expect((term.data as { hm: string }).hm).not.toBe(stored.data.email__hmac) + expect((term.data as { hm: string }).hm).not.toBe( + storedAttrs(stored.data).email__hmac, + ) }) it('is deterministic across separate encryptions of the same value', async () => { @@ -286,9 +319,13 @@ describe('the __hmac key-condition path', () => { if (first.failure) throw new Error(first.failure.message) if (second.failure) throw new Error(second.failure.message) - expect(first.data.email__hmac).toBe(second.data.email__hmac) + expect(storedAttrs(first.data).email__hmac).toBe( + storedAttrs(second.data).email__hmac, + ) // ...while the ciphertext itself is not deterministic. - expect(first.data.email__source).not.toBe(second.data.email__source) + expect(storedAttrs(first.data).email__source).not.toBe( + storedAttrs(second.data).email__source, + ) }) }) diff --git a/packages/stack/__tests__/dynamodb/helpers.test.ts b/packages/stack/__tests__/dynamodb/helpers.test.ts index 3c664237..85ce56cf 100644 --- a/packages/stack/__tests__/dynamodb/helpers.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers.test.ts @@ -25,7 +25,7 @@ const users = encryptedTable('users', { email: encryptedColumn('email').equality(), name: encryptedColumn('name'), blob: encryptedColumn('blob').dataType('json'), - doc: encryptedColumn('doc').dataType('json').searchableJson('users/doc'), + doc: encryptedColumn('doc').dataType('json').searchableJson(), example: { protected: encryptedField('example.protected'), }, diff --git a/packages/stack/__tests__/dynamodb/properties.test.ts b/packages/stack/__tests__/dynamodb/properties.test.ts new file mode 100644 index 00000000..89b8cf7f --- /dev/null +++ b/packages/stack/__tests__/dynamodb/properties.test.ts @@ -0,0 +1,525 @@ +/** + * Property-based coverage (fast-check) for the DynamoDB attribute mapping. + * + * The example-based suites (`helpers.test.ts`, `helpers-v3.test.ts`) pin the + * mapping pointwise. Four data-integrity bugs still slipped through all of + * them, because every one needed a *shape the examples never wrote down*: a + * plaintext key that happens to collide with an EQL envelope key, an attribute + * that happens to end in a storage suffix, a `Date` in a cloned model, a column + * whose JS property name differs from its DB name. These are the cross-cutting + * invariants that make those shapes unrepresentable rather than merely untested: + * + * 1. ROUND TRIP: for any column set and any ciphertext, + * `toItemWithEqlPayloads(toEncryptedDynamoItem(x))` preserves every + * ciphertext and rebuilds the envelope the FFI will accept — scalar + * (untagged) and ste_vec (`k: 'sv'`) alike. + * 2. PASSTHROUGH: an item with NO encrypted payloads is returned unchanged by + * BOTH directions, at any nesting depth, for any attribute name — + * including names that collide with envelope keys (`c`, `hm`, `k`, `sv`, + * `v`, `i`) and with the storage suffixes (`legit__hmac`, `img__source`). + * 3. NO ATTRIBUTE LOSS: neither direction ever reduces the top-level key + * count of an all-plaintext item. + * 4. deepClone is identity-preserving, `Date`-preserving, and shares no + * object reference with its source. + * 5. VERSION FOLLOWS THE TABLE: a v3 table always yields `v: 3` and never + * `k: 'ct'`; a v2 table always yields `v: 2` with `k: 'ct'`. + * 6. NO ENVELOPE METADATA LEAK: a split scalar writes only `__source` + * (plus `__hmac` when `hm` is present) and never leaks `k`/`v`/`i` + * into a stored attribute value. + * 7. PROPERTY vs DB NAME: the stored attribute is keyed by the JS PROPERTY + * name while the rebuilt identifier `i.c` is the DB name. + * + * Every property here is pure — no client, no credentials, no network. The one + * live property (a ZeroKMS `decryptModel(encryptModel(x))` round trip) lives in + * `encrypted-dynamodb-v3.test.ts` with the rest of the live suite. + */ +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' +import { + ciphertextAttrSuffix, + deepClone, + isV3Table, + searchTermAttrSuffix, + toEncryptedDynamoItem, + toItemWithEqlPayloads, +} from '@/dynamodb/helpers' +import { encryptedTable, types } from '@/eql/v3' +import { + encryptedColumn, + encryptedField, + encryptedTable as encryptedTableV2, +} from '@/schema' + +// --------------------------------------------------------------------------- +// Generators +// --------------------------------------------------------------------------- + +/** + * `EncryptedTable.build()` THROWS on duplicate DB names and `encryptedTable()` + * throws on a column name that shadows a table member, so generated schemas are + * kept inside a charset that can produce neither: lowercase, max 8 chars (too + * short to contain a `__source`/`__hmac` suffix), minus the one reserved word + * the charset can reach. + */ +const RESERVED_COLUMN_NAMES = new Set(['build']) + +/** The plaintext attribute the payload properties carry alongside the columns. */ +const PLAINTEXT_KEY = 'zz_plain' + +const safeName = fc + .stringMatching(/^[a-z][a-z0-9_]{0,7}$/) + .filter((n) => !RESERVED_COLUMN_NAMES.has(n) && n !== PLAINTEXT_KEY) + +/** Ciphertexts are never empty in practice, and the write path tests truthiness. */ +const ciphertext = fc.string({ minLength: 1 }) + +/** ste_vec entries, as the FFI emits them for a JSON document. */ +const steVecEntries = fc.array( + fc.record({ s: fc.string(), c: fc.string({ minLength: 1 }) }), + { minLength: 1, maxLength: 3 }, +) + +/** + * Attribute names chosen to be maximally hostile to the mapping's heuristics: + * six that collide with EQL envelope keys, two with the storage suffixes. + */ +const hostileKey = fc.oneof( + fc.constantFrom('c', 'hm', 'k', 'sv', 'v', 'i', 'legit__hmac', 'img__source'), + safeName, +) + +/** An all-plaintext attribute tree: no value anywhere is an EQL payload. */ +const plaintextValue = fc.letrec<{ node: unknown; leaf: unknown }>((tie) => ({ + leaf: fc.oneof( + fc.string(), + fc.integer(), + fc.boolean(), + fc.constant(null), + fc.double({ noNaN: true }), + ), + node: fc.oneof( + { maxDepth: 3, depthSize: 'small' }, + tie('leaf'), + fc.array(tie('node'), { maxLength: 3 }), + fc.dictionary(hostileKey, tie('node'), { maxKeys: 4 }), + ), +})).node + +const plaintextItem = fc.dictionary(hostileKey, plaintextValue, { maxKeys: 5 }) + +/** + * Would the mapping's payload detector see an envelope anywhere in this tree? + * + * The detector is a heuristic — `v` + `i` + (`c` | `sv`) — so an all-plaintext + * object that happens to carry exactly those three keys is genuinely + * indistinguishable from a payload and is *supposed* to be rewritten. The + * passthrough properties are about everything else, so those cases are + * discarded rather than asserted (documenting the heuristic's exact blind spot). + */ +function looksLikeEnvelope(value: unknown): boolean { + if (value === null || typeof value !== 'object') return false + if (Array.isArray(value)) return value.some(looksLikeEnvelope) + const hasEnvelopeKeys = + 'v' in value && 'i' in value && ('c' in value || 'sv' in value) + return hasEnvelopeKeys || Object.values(value).some(looksLikeEnvelope) +} + +// --------------------------------------------------------------------------- +// 1. ROUND TRIP +// --------------------------------------------------------------------------- + +/** A generated v3 schema plus one payload per column. */ +const roundTripCase = fc + .uniqueArray(safeName, { minLength: 1, maxLength: 5 }) + .chain((names) => + fc.record({ + names: fc.constant(names), + kinds: fc.tuple( + ...names.map(() => fc.constantFrom('scalar' as const, 'json' as const)), + ), + cts: fc.tuple(...names.map(() => ciphertext)), + svs: fc.tuple(...names.map(() => steVecEntries)), + hms: fc.tuple( + ...names.map(() => + fc.option(fc.string({ minLength: 1 }), { nil: undefined }), + ), + ), + plain: fc.string(), + }), + ) + +describe('property: write → read round trip', () => { + it('preserves every ciphertext and rebuilds the envelope, for any column set', () => { + fc.assert( + fc.property(roundTripCase, ({ names, kinds, cts, svs, hms, plain }) => { + const table = encryptedTable( + 'users', + Object.fromEntries( + names.map((n, idx) => [ + n, + kinds[idx] === 'json' ? types.Json(n) : types.TextEq(n), + ]), + ), + ) + const encryptedAttrs = Object.keys(table.buildColumnKeyMap()) + + const item: Record = { [PLAINTEXT_KEY]: plain } + const expectedStored: Record = { + [PLAINTEXT_KEY]: plain, + } + const expectedRebuilt: Record = { + [PLAINTEXT_KEY]: plain, + } + + names.forEach((name, idx) => { + const i = { t: 'users', c: name } + if (kinds[idx] === 'json') { + item[name] = { v: 3, k: 'sv', i, sv: svs[idx] } + expectedStored[`${name}${ciphertextAttrSuffix}`] = svs[idx] + expectedRebuilt[name] = { + i: { c: name, t: 'users' }, + v: 3, + k: 'sv', + sv: svs[idx], + } + return + } + item[name] = { + v: 3, + i, + c: cts[idx], + ...(hms[idx] === undefined ? {} : { hm: hms[idx] }), + } + expectedStored[`${name}${ciphertextAttrSuffix}`] = cts[idx] + if (hms[idx] !== undefined) { + expectedStored[`${name}${searchTermAttrSuffix}`] = hms[idx] + } + // The search term is a write-side index, not part of the envelope: + // the read path drops it rather than round-tripping it. + expectedRebuilt[name] = { + i: { c: name, t: 'users' }, + v: 3, + c: cts[idx], + } + }) + + const stored = toEncryptedDynamoItem(item, encryptedAttrs) + expect(stored).toEqual(expectedStored) + expect(toItemWithEqlPayloads(stored, table)).toEqual(expectedRebuilt) + }), + ) + }) +}) + +// --------------------------------------------------------------------------- +// 2. PASSTHROUGH — pins two of the four review bugs +// --------------------------------------------------------------------------- + +describe('property: plaintext passthrough (both directions)', () => { + const users = encryptedTable('users', { + email: types.TextEq('email'), + note: types.Text('note'), + meta: types.Json('meta'), + }) + const encryptedAttrs = Object.keys(users.buildColumnKeyMap()) + + it('the write path returns an item with no payloads unchanged, at any depth', () => { + // BUG PINNED: a nested plaintext object with a `c` key was mistaken for a + // ciphertext, rewritten to `__source`, and every sibling key + // DISCARDED — silent data loss on a PutCommand. + fc.assert( + fc.property(plaintextItem, (item) => { + fc.pre(!looksLikeEnvelope(item)) + expect(toEncryptedDynamoItem(item, encryptedAttrs)).toEqual(item) + }), + { numRuns: 1000 }, + ) + }) + + it('the read path returns an item with no envelopes unchanged, at any depth', () => { + // BUG PINNED: ANY attribute ending in `__hmac` was dropped on read, so a + // customer's own `signature__hmac` vanished; and any nested `*__source` was + // handed to the FFI as if it were a ciphertext. + fc.assert( + fc.property(plaintextItem, (item) => { + expect(toItemWithEqlPayloads(item, users)).toEqual(item) + }), + { numRuns: 1000 }, + ) + }) + + // --------------------------------------------------------------------------- + // 3. NO ATTRIBUTE LOSS + // --------------------------------------------------------------------------- + + it('neither direction reduces the top-level key count of an all-plaintext item', () => { + fc.assert( + fc.property(plaintextItem, (item) => { + fc.pre(!looksLikeEnvelope(item)) + const keys = Object.keys(item) + const written = toEncryptedDynamoItem(item, encryptedAttrs) + const read = toItemWithEqlPayloads(item, users) + + expect(Object.keys(written).length).toBeGreaterThanOrEqual(keys.length) + expect(Object.keys(read).length).toBeGreaterThanOrEqual(keys.length) + for (const key of keys) { + expect(written).toHaveProperty([key]) + expect(read).toHaveProperty([key]) + } + }), + { numRuns: 1000 }, + ) + }) +}) + +// --------------------------------------------------------------------------- +// 4. deepClone +// --------------------------------------------------------------------------- + +/** Every object/array reachable from `value`, cycle-safe. */ +function objectRefs(value: unknown, seen = new Set()): Set { + if (value === null || typeof value !== 'object') return seen + if (seen.has(value)) return seen + seen.add(value) + for (const child of Object.values(value)) objectRefs(child, seen) + return seen +} + +describe('property: deepClone', () => { + // BUG PINNED: the previous hand-rolled `Object.entries` reduce flattened every + // non-plain object to `{}`, silently destroying `Date` values and making the + // whole `types.Timestamp*` / `types.Date*` domain family unusable here. + it('is identity-preserving over arbitrary JSON documents', () => { + fc.assert( + fc.property(fc.jsonValue(), (value) => { + expect(deepClone(value)).toEqual(value) + }), + ) + }) + + it('preserves a Date as a real Date with the same time', () => { + fc.assert( + fc.property(fc.date({ noInvalidDate: true }), (at) => { + const cloned = deepClone({ at, nested: { at } }) + expect(cloned.at).toBeInstanceOf(Date) + expect(cloned.at.getTime()).toBe(at.getTime()) + expect(cloned.nested.at).toBeInstanceOf(Date) + expect(cloned.nested.at.getTime()).toBe(at.getTime()) + }), + ) + }) + + it('shares no object reference with its source', () => { + fc.assert( + fc.property( + fc.dictionary(fc.string(), fc.jsonValue(), { minKeys: 1 }), + (value) => { + const cloned = deepClone(value) + const sourceRefs = objectRefs(value) + for (const ref of objectRefs(cloned)) { + expect(sourceRefs.has(ref)).toBe(false) + } + }, + ), + ) + }) +}) + +// --------------------------------------------------------------------------- +// 5. VERSION FOLLOWS THE TABLE +// --------------------------------------------------------------------------- + +describe('property: the rebuilt wire version follows the table', () => { + it('a v3 table always yields v: 3 and never a k: "ct" tag', () => { + fc.assert( + fc.property(safeName, ciphertext, (name, ct) => { + const table = encryptedTable('t', { [name]: types.TextEq(name) }) + expect(isV3Table(table)).toBe(true) + + const rebuilt = toItemWithEqlPayloads( + { [`${name}${ciphertextAttrSuffix}`]: ct }, + table, + )[name] as Record + + expect(rebuilt.v).toBe(3) + expect(rebuilt).not.toHaveProperty('k') + expect(rebuilt.c).toBe(ct) + }), + ) + }) + + it('a v3 JSON column keeps v: 3 with the mandatory k: "sv"', () => { + fc.assert( + fc.property(safeName, steVecEntries, (name, entries) => { + const table = encryptedTable('t', { [name]: types.Json(name) }) + + const rebuilt = toItemWithEqlPayloads( + { [`${name}${ciphertextAttrSuffix}`]: entries }, + table, + )[name] as Record + + expect(rebuilt.v).toBe(3) + expect(rebuilt.k).toBe('sv') + expect(rebuilt.sv).toEqual(entries) + }), + ) + }) + + it('a v2 table always yields v: 2 with k: "ct"', () => { + fc.assert( + fc.property(safeName, ciphertext, (name, ct) => { + const table = encryptedTableV2('t', { + [name]: encryptedColumn(name).equality(), + }) + expect(isV3Table(table)).toBe(false) + + const rebuilt = toItemWithEqlPayloads( + { [`${name}${ciphertextAttrSuffix}`]: ct }, + table, + )[name] as Record + + expect(rebuilt.v).toBe(2) + expect(rebuilt.k).toBe('ct') + expect(rebuilt.c).toBe(ct) + }), + ) + }) + + it('a v2 table with a grouped field still yields v: 2 with k: "ct"', () => { + fc.assert( + fc.property(safeName, safeName, ciphertext, (group, leaf, ct) => { + const table = encryptedTableV2('t', { + [group]: { [leaf]: encryptedField(`${group}.${leaf}`) }, + }) + expect(isV3Table(table)).toBe(false) + + const rebuilt = toItemWithEqlPayloads( + { [group]: { [`${leaf}${ciphertextAttrSuffix}`]: ct } }, + table, + )[group] as Record> + + expect(rebuilt[leaf]?.v).toBe(2) + expect(rebuilt[leaf]?.k).toBe('ct') + }), + ) + }) +}) + +// --------------------------------------------------------------------------- +// 6. NO ENVELOPE METADATA LEAK +// --------------------------------------------------------------------------- + +/** Does any value in this tree carry an envelope metadata key? */ +function hasEnvelopeMetadata(value: unknown): boolean { + if (value === null || typeof value !== 'object') return false + if (Array.isArray(value)) return value.some(hasEnvelopeMetadata) + if ('k' in value || 'v' in value || 'i' in value) return true + return Object.values(value).some(hasEnvelopeMetadata) +} + +describe('property: no envelope metadata reaches storage', () => { + it('a split scalar writes only __source (and __hmac when hm is present)', () => { + fc.assert( + fc.property( + safeName, + ciphertext, + fc.option(fc.string({ minLength: 1 }), { nil: undefined }), + // Terms with no DynamoDB query surface: they must be dropped, not stored. + fc.record({ + op: fc.option(fc.string(), { nil: undefined }), + ob: fc.option(fc.array(fc.string()), { nil: undefined }), + bf: fc.option(fc.array(fc.integer()), { nil: undefined }), + }), + (name, ct, hm, noise) => { + const table = encryptedTable('t', { [name]: types.TextEq(name) }) + const stored = toEncryptedDynamoItem( + { + [name]: { + v: 3, + i: { t: 't', c: name }, + c: ct, + ...(hm === undefined ? {} : { hm }), + ...noise, + }, + }, + Object.keys(table.buildColumnKeyMap()), + ) + + const expectedKeys = [`${name}${ciphertextAttrSuffix}`] + if (hm !== undefined) { + expectedKeys.push(`${name}${searchTermAttrSuffix}`) + } + expect(Object.keys(stored).sort()).toEqual(expectedKeys.sort()) + expect(stored[`${name}${ciphertextAttrSuffix}`]).toBe(ct) + for (const value of Object.values(stored)) { + expect(hasEnvelopeMetadata(value)).toBe(false) + } + }, + ), + ) + }) + + it('a ste_vec document is stored as its bare sv array', () => { + fc.assert( + fc.property(safeName, steVecEntries, (name, entries) => { + const table = encryptedTable('t', { [name]: types.Json(name) }) + const stored = toEncryptedDynamoItem( + { + [name]: { v: 3, k: 'sv', i: { t: 't', c: name }, sv: entries }, + }, + Object.keys(table.buildColumnKeyMap()), + ) + + expect(Object.keys(stored)).toEqual([`${name}${ciphertextAttrSuffix}`]) + expect(stored[`${name}${ciphertextAttrSuffix}`]).toEqual(entries) + expect(hasEnvelopeMetadata(stored)).toBe(false) + }), + ) + }) +}) + +// --------------------------------------------------------------------------- +// 7. PROPERTY vs DB NAME — the most severe bug found in review +// --------------------------------------------------------------------------- + +describe('property: attributes are keyed by property name, identified by DB name', () => { + // BUG PINNED: the two names were conflated. A column declared + // `emailAddress: types.TextEq('email_address')` was matched against the DB + // name, so the attribute was never split on write, its `__hmac` was never + // emitted (killing every key-condition query on it), and on read the mapping + // recursed into the payload instead of rebuilding it. + const distinctNames = fc + .tuple(safeName, safeName) + .filter(([property, dbName]) => property !== dbName) + + it('splits under the property name and rebuilds i.c as the DB name', () => { + fc.assert( + fc.property( + distinctNames, + ciphertext, + fc.string({ minLength: 1 }), + ([property, dbName], ct, hm) => { + const table = encryptedTable('t', { + [property]: types.TextEq(dbName), + }) + + const stored = toEncryptedDynamoItem( + { [property]: { v: 3, i: { t: 't', c: dbName }, c: ct, hm } }, + Object.keys(table.buildColumnKeyMap()), + ) + + // Keyed by the PROPERTY name — the DB name never appears in storage. + expect(stored).toEqual({ + [`${property}${ciphertextAttrSuffix}`]: ct, + [`${property}${searchTermAttrSuffix}`]: hm, + }) + + // Identified by the DB name — this is what the FFI looks up. + expect(toItemWithEqlPayloads(stored, table)).toEqual({ + [property]: { i: { c: dbName, t: 't' }, v: 3, c: ct }, + }) + }, + ), + ) + }) +}) diff --git a/packages/stack/src/dynamodb/index.ts b/packages/stack/src/dynamodb/index.ts index 1f11764f..e415be06 100644 --- a/packages/stack/src/dynamodb/index.ts +++ b/packages/stack/src/dynamodb/index.ts @@ -64,7 +64,13 @@ export function encryptedDynamoDB( ): EncryptedDynamoDBInstance { const { encryptionClient, options } = config - return { + // The public interface is OVERLOADED per wire version (a v3 overload that + // knows the storage split, a v2 one that does not). A single generic + // implementation signature cannot be inferred as satisfying both arms, so the + // instance is built against the erased shape and asserted once, here. Nothing + // about the runtime differs between the overloads — the table decides + // everything, and it does so at runtime. + const instance = { encryptModel>( item: T, table: AnyEncryptedTable, @@ -113,9 +119,16 @@ export function encryptedDynamoDB( ) }, } + + return instance as EncryptedDynamoDBInstance } +export type { AuditConfig } from './operations/base-operation' export type { + AnyEncryptedTable, + DecryptedAttributes, + DynamoDBEncryptionClient, + EncryptedAttributes, EncryptedDynamoDBConfig, EncryptedDynamoDBError, EncryptedDynamoDBInstance, diff --git a/packages/stack/src/dynamodb/types.ts b/packages/stack/src/dynamodb/types.ts index 4cdff90a..5bcf8835 100644 --- a/packages/stack/src/dynamodb/types.ts +++ b/packages/stack/src/dynamodb/types.ts @@ -1,8 +1,16 @@ import type { ProtectErrorCode } from '@cipherstash/protect-ffi' import type { EncryptionClient } from '@/encryption' -import type { AnyV3Table } from '@/eql/v3' +import type { + AnyV3Table, + EncryptedTable as EncryptedV3Table, + InferPlaintext, + PlaintextForColumn, + QueryTypesForColumn, + V3ModelInput, +} from '@/eql/v3' import type { EncryptedTable, EncryptedTableColumn } from '@/schema' import type { EncryptedValue } from '@/types' +import type { ciphertextAttrSuffix, searchTermAttrSuffix } from './helpers' import type { BulkDecryptModelsOperation } from './operations/bulk-decrypt-models' import type { BulkEncryptModelsOperation } from './operations/bulk-encrypt-models' import type { DecryptModelOperation } from './operations/decrypt-model' @@ -39,8 +47,8 @@ export type AnyEncryptedTable = export type DynamoDBEncryptionClient = { encryptModel(input: never, table: never): unknown bulkEncryptModels(input: never, table: never): unknown - decryptModel(input: never, table?: never): unknown - bulkDecryptModels(input: never, table?: never): unknown + decryptModel(input: never, table: never): unknown + bulkDecryptModels(input: never, table: never): unknown } type ChainableEncryptOperation = { @@ -105,24 +113,174 @@ export interface EncryptedDynamoDBError extends Error { details?: Record } +// --------------------------------------------------------------------------- +// The DynamoDB storage split, at the type level +// --------------------------------------------------------------------------- + +/** The `__source` / `__hmac` suffixes, read off the runtime constants so the + * types cannot drift from the mapping in `helpers.ts`. */ +type CiphertextSuffix = typeof ciphertextAttrSuffix +type SearchTermSuffix = typeof searchTermAttrSuffix + +/** The column map a v3 table was declared with. */ +type V3Columns = + Table extends EncryptedV3Table ? C : never + +/** + * What `toEncryptedDynamoItem` writes into `__source` for a column: the + * `sv` array for a JSON document, the base64 ciphertext `c` for every scalar. + */ +type SourceAttribute = + 'searchableJson' extends QueryTypesForColumn ? unknown[] : string + +/** + * Does this column mint the `hm` term that becomes `__hmac`? + * + * Mirrors `indexesForCapabilities` (eql/v3/columns.ts) exactly: `hm` comes from + * the `unique` index, which is emitted when a domain is equality-capable AND is + * either not an ordering domain or is text (text equality is always HMAC-based, + * numeric/date ordering domains answer equality via their ordering term and + * emit no `unique`). A JSON document keeps its terms inside `sv`, so it has no + * separate search-term attribute. + * + * Derived from the public `QueryTypesForColumn` / `PlaintextForColumn` rather + * than the internal domain literal, so it stays inside the v3 barrel's API. + */ +type HasSearchTerm = + 'searchableJson' extends QueryTypesForColumn + ? false + : 'equality' extends QueryTypesForColumn + ? 'orderAndRange' extends QueryTypesForColumn + ? [PlaintextForColumn] extends [string] + ? true + : false + : true + : false + +/** Flatten an intersection into a single object type for readable errors. */ +type Simplify = { [K in keyof T]: T[K] } + +/** + * The DynamoDB attribute map `encryptModel` actually returns for a v3 table. + * + * A declared column `email` does NOT survive as `email`: the adapter deletes it + * and writes `email__source` (plus `email__hmac` for equality domains). Typing + * the result as the input model — what the v2 overload still does — is a lie + * that type-checks `result.data.email` (always `undefined` at runtime) and + * rejects `result.data.email__source` (the value you actually want). + * + * Keys that name no column pass through untouched — partition/sort keys, GSI + * attributes, anything else on the item. + * + * LIMITATION: a v3 column declared under a dotted path (`'profile.ssn'`) is + * split *inside* the nested `profile` map at runtime. The model key is + * `profile`, not `profile.ssn`, so it passes through here unchanged and the + * nested split is not modelled. + */ +export type EncryptedAttributes
= Simplify< + { + [K in keyof T as K extends keyof V3Columns
& string + ? `${K}${CiphertextSuffix}` + : K]: K extends keyof V3Columns
+ ? SourceAttribute[K]> + : T[K] + } & { + // Optional: the term is only written when the encrypted value produced one, + // so a null/absent field leaves the attribute off the item entirely. + [K in keyof T as K extends keyof V3Columns
& string + ? HasSearchTerm[K]> extends true + ? `${K}${SearchTermSuffix}` + : never + : never]?: string + } +> + +/** + * The inverse of {@link EncryptedAttributes}: the plaintext model + * `decryptModel` returns for an item read back out of DynamoDB. + * + * `__source` folds back to `col` with the column's plaintext type, + * `__hmac` is dropped (it is a query term, not data), and every other + * attribute passes through. Declared this way — rather than taking the + * plaintext model as the input parameter — so `T` is inferred from the argument + * a caller actually has: the stored attribute map. + */ +export type DecryptedAttributes
= Simplify<{ + [K in keyof T as K extends `${infer Base}${SearchTermSuffix}` + ? Base extends keyof V3Columns
+ ? never + : K + : K extends `${infer Base}${CiphertextSuffix}` + ? Base extends keyof V3Columns
+ ? Base + : K + : K]: K extends `${infer Base}${CiphertextSuffix}` + ? Base extends keyof InferPlaintext
+ ? InferPlaintext
[Base] + : T[K] + : T[K] +}> + export interface EncryptedDynamoDBInstance { + /** + * EQL v3: the input model is checked against the table's column types, and + * the result is the {@link EncryptedAttributes} storage split. + */ + encryptModel
>( + item: V3ModelInput, + table: Table, + ): EncryptModelOperation> + /** + * EQL v2. Unchanged, so existing callers keep compiling — v2 columns do not + * carry their index configuration in the type, so the storage split cannot be + * derived. The returned `T` is the INPUT model, not what is on the wire; read + * `__source` / `__hmac` through a type of your own. + */ encryptModel>( item: T, - table: AnyEncryptedTable, + table: EncryptedTable, ): EncryptModelOperation + /** EQL v3. See {@link EncryptedDynamoDBInstance.encryptModel}. */ + bulkEncryptModels< + Table extends AnyV3Table, + T extends Record, + >( + items: Array>, + table: Table, + ): BulkEncryptModelsOperation> + /** EQL v2. See {@link EncryptedDynamoDBInstance.encryptModel}. */ bulkEncryptModels>( items: T[], - table: AnyEncryptedTable, + table: EncryptedTable, ): BulkEncryptModelsOperation + /** + * EQL v3: `item` is the stored attribute map (`__source` / + * `__hmac`), and the result is the {@link DecryptedAttributes} plaintext + * model it folds back to. + */ + decryptModel
>( + item: T, + table: Table, + ): DecryptModelOperation> + /** EQL v2. Unchanged. */ decryptModel>( item: Record, - table: AnyEncryptedTable, + table: EncryptedTable, ): DecryptModelOperation + /** EQL v3. See {@link EncryptedDynamoDBInstance.decryptModel}. */ + bulkDecryptModels< + Table extends AnyV3Table, + T extends Record, + >( + items: T[], + table: Table, + ): BulkDecryptModelsOperation> + /** EQL v2. Unchanged. */ bulkDecryptModels>( items: Record[], - table: AnyEncryptedTable, + table: EncryptedTable, ): BulkDecryptModelsOperation } diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index bc9a3a1a..2dca84cf 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -453,15 +453,32 @@ const dynamo = encryptedDynamoDB({ `table` is either an EQL v3 table (`types.*` domains) or an EQL v2 one. -| Method | Signature | Returns | +Each method is overloaded on the table's EQL version. + +**EQL v3** — the item is checked against the table's column domains, and the result is typed as the attribute map that is actually stored: a declared column `email` becomes `email__source` (plus `email__hmac` if its domain mints one), NOT `email`. + +| Method | Signature | Resolves to | |---|---|---| -| `encryptModel` | `(item: T, table)` | `EncryptModelOperation` | -| `bulkEncryptModels` | `(items: T[], table)` | `BulkEncryptModelsOperation` | -| `decryptModel` | `(item: Record, table)` | `DecryptModelOperation` (resolves to `Decrypted`) | -| `bulkDecryptModels` | `(items: Record[], table)` | `BulkDecryptModelsOperation` (resolves to `Decrypted[]`) | +| `encryptModel` | `(item, v3Table)` | `EncryptedAttributes` | +| `bulkEncryptModels` | `(items, v3Table)` | `EncryptedAttributes[]` | +| `decryptModel` | `(storedItem, v3Table)` | `DecryptedAttributes` — `__source` folded back to the column, `__hmac` dropped | +| `bulkDecryptModels` | `(storedItems, v3Table)` | `DecryptedAttributes[]` | + +Let `T` be inferred from the argument; do not pass explicit type arguments on the v3 path. + +**EQL v2** — unchanged. `T` is the model you pass in, so the result type does NOT reflect the `__source` / `__hmac` split; declare a type of your own to read those attributes. + +| Method | Signature | Resolves to | +|---|---|---| +| `encryptModel` | `(item: T, v2Table)` | `T` | +| `bulkEncryptModels` | `(items: T[], v2Table)` | `T[]` | +| `decryptModel` | `(item: Record, v2Table)` | `T` | +| `bulkDecryptModels` | `(items: Record[], v2Table)` | `T[]` | All operations are thenable (awaitable) and support `.audit({ metadata })` chaining. See the note in Setup about decrypt-side audit metadata and the `EncryptionV3` client. +Types exported from `@cipherstash/stack/dynamodb`: `EncryptedDynamoDBInstance`, `EncryptedDynamoDBConfig`, `EncryptedDynamoDBError`, `AnyEncryptedTable`, `DynamoDBEncryptionClient`, `EncryptedAttributes`, `DecryptedAttributes`, `AuditConfig`. + ### Querying Encrypted Attributes Use the encryption client directly (not the DynamoDB helper): From 7105c5ca395091b3e252f63f4d1f3e464945bcf4 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 21 Jul 2026 09:57:18 +1000 Subject: [PATCH 10/20] fix(stack): second-review fixes for DynamoDB v3 adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A four-dimension review (architecture / types / tests / v2-v3 parity) of the completed branch. The parity audit found no accidental divergences — every v2/v3 difference is intentional and traces to the wire format or the FFI. The test review found one confirmed HIGH bug; the rest are hardening. HIGH — leaf-fallback over-match (read path). `matchColumn` fell back from the dotted path to the bare leaf, which is needed for v2 `encryptedField('amount')` groups but wrongly fired for v3: a nested `note__source` matched a same-named TOP-LEVEL `note` column and rewrote a plaintext sibling as an envelope, handing it to the FFI as a decrypt target. v3 always registers full dotted paths, so it never needs the fallback — scoped it to nested v2 attributes only. Regression test plus dotted-path and 3-level-nesting tests added. MEDIUM — decrypt path rebuilt the table config + column map once PER ITEM (`toItemWithEqlPayloads` called `build()` and `resolveEncryptColumnMap` internally), asymmetric with the encrypt path which resolves once. Extracted `buildReadContext`; bulk-decrypt now builds it once and passes it, O(N)→O(1). Audit-on-decrypt is a silent no-op with the typed EncryptionV3 client (it has no decrypt audit surface). Both the parity and architecture reviews flagged the silence; `resolveDecryptResult` now logs a debug line when metadata is dropped, so it is observable. Behaviour unchanged. Tests: - resolve-decrypt.test.ts — pure coverage for `resolveDecryptResult` (both client shapes) and `throwPreservingCode`, previously only exercised live. - client-compat.test-d.ts — bulk return-type storage split now type-locked (was callability-only), plus a second wrong-plaintext-type rejection. - all-or-nothing bulk decrypt semantics pinned (one bad item fails the batch). Not changed, with reasons: the passthrough property is not widened to the collision case — a top-level `__source` is legitimately the storage form of that column, so it cannot be folded into a passthrough oracle; the direct regression test is the correct pin. The v2 overload return type and the nested dotted-path type gap remain documented limitations (runtime-correct). A DynamoDB-Local / marshalling integration test remains a follow-up (needs an AWS SDK dev dependency decision). 1015 passing overall; dynamodb suite 68 pure + type + live. --- .../dynamodb/client-compat.test-d.ts | 26 +++++++ .../dynamodb/encrypted-dynamodb.test.ts | 13 ++++ .../__tests__/dynamodb/helpers-v3.test.ts | 41 +++++++++++ .../dynamodb/resolve-decrypt.test.ts | 67 +++++++++++++++++ packages/stack/src/dynamodb/helpers.ts | 71 +++++++++++++++---- .../operations/bulk-decrypt-models.ts | 6 +- 6 files changed, 210 insertions(+), 14 deletions(-) create mode 100644 packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts diff --git a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts index 7f6b54a2..ae879dce 100644 --- a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts +++ b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts @@ -155,9 +155,35 @@ describe('the v3 overload types the DynamoDB storage split', () => { expectTypeOf(result.data).not.toHaveProperty('meta__hmac') }) + it('applies the same storage split to the bulk return type', async () => { + const result = await dynamo.bulkEncryptModels( + [{ pk: 'a', email: 'a@b.com', age: 3 }], + usersV3, + ) + if (result.failure) return + + expectTypeOf(result.data[0].email__source).toEqualTypeOf() + expectTypeOf(result.data[0].email__hmac).toEqualTypeOf() + expectTypeOf(result.data[0].age__source).toEqualTypeOf() + expectTypeOf(result.data[0]).not.toHaveProperty('email') + }) + + it('folds bulk stored attributes back to plaintext on decrypt', async () => { + const result = await dynamo.bulkDecryptModels( + [{ pk: 'a', email__source: 'ct', email__hmac: 'hm' }], + usersV3, + ) + if (result.failure) return + + expectTypeOf(result.data[0].email).toEqualTypeOf() + expectTypeOf(result.data[0]).not.toHaveProperty('email__source') + }) + it('rejects a field whose type does not match its column domain', () => { // @ts-expect-error - `email` is a text domain, not a number dynamo.encryptModel({ pk: 'a', email: 42 }, usersV3) + // @ts-expect-error - `age` is an integer domain, not a string + dynamo.encryptModel({ pk: 'a', age: 'not-a-number' }, usersV3) }) it('folds the stored attributes back to plaintext on decrypt', async () => { diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts index 78bdd3bb..3b7d8f51 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts @@ -419,6 +419,19 @@ describe('error handling', () => { expect(result.failure?.details).toEqual({ context: 'bulkDecryptModels' }) }) + it('fails the whole batch when one item is undecryptable (all-or-nothing)', async () => { + const result = await dynamo.bulkDecryptModels( + // One malformed item alongside a well-formed plaintext-only one. + [{ email__source: 'not-a-ciphertext' }, { pk: 'ok' }], + users, + ) + + // The batch is all-or-nothing: a single bad item fails the whole call, and + // no partial data is returned. + expect(result.failure).toBeDefined() + expect((result as { data?: unknown }).data).toBeUndefined() + }) + it('routes failures to the configured errorHandler', async () => { const seen: { code: string; message: string }[] = [] const instrumented = encryptedDynamoDB({ diff --git a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts index 35d2bc90..144a1892 100644 --- a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts @@ -229,6 +229,47 @@ describe('regressions found in review', () => { expect(toItemWithEqlPayloads(item, users)).toEqual(item) }) + it('does not rebuild a nested __source whose leaf collides with a top-level column', () => { + // `note` is a TOP-LEVEL column; there is no `profile.note`. A v3 table + // registers full dotted paths, so a nested `note__source` must NOT match + // the top-level `note` by bare leaf — doing so rewrote a plaintext sibling + // as an envelope and handed it to the FFI as a decrypt target. + const t = encryptedTable('t', { + email: types.TextEq('email'), + note: types.Text('note'), + }) + const item = { profile: { note__source: 'plaintext, not a ciphertext' } } + + expect(toItemWithEqlPayloads(item, t)).toEqual(item) + }) + + it('still rebuilds a v3 column declared with a dotted path', () => { + const t = encryptedTable('t', { + 'profile.ssn': types.TextEq('profile.ssn'), + }) + + expect( + toItemWithEqlPayloads({ profile: { ssn__source: 'CT' } }, t), + ).toEqual({ + profile: { ssn: { i: { c: 'profile.ssn', t: 't' }, v: 3, c: 'CT' } }, + }) + }) + + it('round-trips a three-level dotted path', () => { + const t = encryptedTable('t', { 'a.b.c': types.TextEq('a.b.c') }) + const stored = toEncryptedDynamoItem( + { + a: { b: { c: { v: 3, i: { t: 't', c: 'a.b.c' }, c: 'CT', hm: 'H' } } }, + }, + Object.keys(t.buildColumnKeyMap()), + ) + + expect(stored).toEqual({ a: { b: { c__source: 'CT', c__hmac: 'H' } } }) + expect(toItemWithEqlPayloads(stored, t)).toEqual({ + a: { b: { c: { i: { c: 'a.b.c', t: 't' }, v: 3, c: 'CT' } } }, + }) + }) + it('identifies a column by its DB name when it differs from the property', () => { // `emailAddress: types.TextEq('email_address')` — matching must happen on // the property name, identification on the DB name. diff --git a/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts b/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts new file mode 100644 index 00000000..27dab582 --- /dev/null +++ b/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts @@ -0,0 +1,67 @@ +/** + * Pure unit tests for the two client-shape helpers that bridge the nominal + * `EncryptionClient` (chainable, carries `.audit()`) and the typed client from + * `EncryptionV3` (plain `Promise`). Both branches were previously only + * reachable through a live ZeroKMS decrypt; these move that assurance onto the + * pure CI lane. No credentials, no network. + */ +import { describe, expect, it } from 'vitest' +import { resolveDecryptResult, throwPreservingCode } from '@/dynamodb/helpers' + +describe('resolveDecryptResult', () => { + it('awaits a plain promise when the operation has no .audit (typed client)', async () => { + const result = await resolveDecryptResult( + Promise.resolve({ data: { x: 1 } }), + { metadata: { ignored: true } }, + ) + + expect(result).toEqual({ data: { x: 1 } }) + }) + + it('chains .audit and forwards metadata when present (nominal client)', async () => { + let seen: unknown + const operation = { + audit(config: { metadata?: Record }) { + seen = config.metadata + return Promise.resolve({ data: { y: 2 } }) + }, + } + + const result = await resolveDecryptResult(operation, { + metadata: { m: 42 }, + }) + + expect(result).toEqual({ data: { y: 2 } }) + expect(seen).toEqual({ m: 42 }) + }) + + it('propagates a failure result unchanged', async () => { + const failure = { failure: { message: 'boom', code: 'X' } } + + await expect( + resolveDecryptResult(Promise.resolve(failure), { metadata: {} }), + ).resolves.toEqual(failure) + }) +}) + +describe('throwPreservingCode', () => { + it('rethrows as an Error carrying the FFI code', () => { + try { + throwPreservingCode({ message: 'boom', code: 'UNKNOWN_COLUMN' }) + expect.unreachable('should have thrown') + } catch (error) { + expect(error).toBeInstanceOf(Error) + expect((error as Error).message).toBe('boom') + expect((error as { code?: string }).code).toBe('UNKNOWN_COLUMN') + } + }) + + it('tolerates a missing code', () => { + try { + throwPreservingCode({ message: 'boom' }) + expect.unreachable('should have thrown') + } catch (error) { + expect((error as { code?: string }).code).toBeUndefined() + } + }) +}) diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index db917714..a7d89e8a 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -104,6 +104,16 @@ export async function resolveDecryptResult( audit?: (data: { metadata?: Record }) => unknown } + if (typeof chainable?.audit !== 'function' && auditData.metadata) { + // The typed EncryptionV3 client returns a plain promise with no decrypt + // audit surface, so the metadata has nowhere to go. Make the drop + // observable rather than silent — use the nominal client for audited + // decrypts. + logger.debug( + 'DynamoDB: decrypt audit metadata ignored — the typed client has no decrypt audit surface; use Encryption({ config: { eqlVersion: 3 } }) for audited decrypts.', + ) + } + const resolved = typeof chainable?.audit === 'function' ? await chainable.audit(auditData) @@ -273,28 +283,63 @@ export function toEncryptedDynamoItem( ) } +/** + * The row-invariant table facts the read path needs. `build()` and + * `resolveEncryptColumnMap` are not memoized, so a bulk decrypt of N items must + * build this ONCE and reuse it — see `buildReadContext`. `columnPaths` are the + * keys a model's fields are MATCHED on (JS property names); `toColumnName` maps + * a match to the name the FFI config is keyed by. For v3 those differ whenever + * a column is declared `emailAddress: types.TextEq('email_address')`. + */ +export type ReadContext = { + isV3: boolean + v: 2 | 3 + encryptConfig: { + tableName: string + columns: Record< + string, + { cast_as?: string; indexes: Record } + > + } + columnPaths: string[] + toColumnName: (path: string) => string +} + +/** Resolve the row-invariant read context for a table, once. */ +export function buildReadContext(schema: AnyEncryptedTable): ReadContext { + const isV3 = isV3Table(schema) + const { columnPaths, toColumnName } = resolveEncryptColumnMap(schema) + return { + isV3, + v: isV3 ? 3 : 2, + encryptConfig: schema.build(), + columnPaths, + toColumnName, + } +} + export function toItemWithEqlPayloads( decrypted: Record, encryptionSchema: AnyEncryptedTable, + // Resolved once here for the single-item path; passed in by the bulk path so + // it is not rebuilt per item. + context: ReadContext = buildReadContext(encryptionSchema), ): Record { - const v = isV3Table(encryptionSchema) ? 3 : 2 - // Hoisted: `build()` is not memoized, and this used to run once per attribute - // per nesting level per item. - const encryptConfig = encryptionSchema.build() - // `columnPaths` are the keys a model's fields are MATCHED on (JS property - // names); `toColumnName` maps a match to the name the FFI config is keyed by. - // For v3 those differ whenever a column is declared as - // `emailAddress: types.TextEq('email_address')` — matching on the config keys - // instead would miss the attribute entirely, drop its `__hmac`, and recurse - // into the payload. Reuses the core resolver rather than re-deriving it. - const { columnPaths, toColumnName } = - resolveEncryptColumnMap(encryptionSchema) + const { isV3, v, encryptConfig, columnPaths, toColumnName } = context /** Resolve an attribute back to the column path it was written from. */ function matchColumn(leaf: string, prefix: string): string | undefined { const dotted = prefix ? `${prefix}.${leaf}` : leaf if (columnPaths.includes(dotted)) return dotted - if (columnPaths.includes(leaf)) return leaf + + // Bare-leaf fallback, v2-only. A v2 `encryptedField('amount')` inside a + // group registers the bare leaf `amount`, so a nested `amount__source` has + // to match it by leaf. v3 always registers the full dotted path + // (`'profile.ssn': types.TextEq('profile.ssn')`), so it never needs this — + // and must NOT use it: a nested `note__source` would otherwise match a + // same-named TOP-LEVEL `note` column and rewrite a plaintext sibling as an + // envelope. Scope the fallback to nested v2 attributes only. + if (!isV3 && prefix && columnPaths.includes(leaf)) return leaf return undefined } diff --git a/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts b/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts index 29daa818..d6d8b8b3 100644 --- a/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts +++ b/packages/stack/src/dynamodb/operations/bulk-decrypt-models.ts @@ -2,6 +2,7 @@ import { type Result, withResult } from '@byteslice/result' import type { Decrypted, EncryptedValue } from '@/types' import { logger } from '@/utils/logger' import { + buildReadContext, handleError, resolveDecryptResult, throwPreservingCode, @@ -43,8 +44,11 @@ export class BulkDecryptModelsOperation< logger.debug(`DynamoDB: bulk decrypting ${this.items.length} models.`) return await withResult( async () => { + // Resolve the table's read context once, not once per item — `build()` + // and the column map are row-invariant. + const readContext = buildReadContext(this.table) const itemsWithEqlPayloads = this.items.map((item) => - toItemWithEqlPayloads(item, this.table), + toItemWithEqlPayloads(item, this.table, readContext), ) const client = this.encryptionClient as CallableEncryptionClient From 6368ce27d9d40c86b4ffda3b55632986e4e6564b Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 21 Jul 2026 10:07:19 +1000 Subject: [PATCH 11/20] docs(skills): fix CodeRabbit findings in stash-dynamodb, pin __hmac domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three documentation findings, all in skills/stash-dynamodb/SKILL.md (which ships to customers and has no automated guard). Verified each against source. F3 (test-driven) — the `__hmac` "only for equality-capable columns" wording (line 39) was imprecise and contradicted the file's own precise domain table: several `*Ord` domains ARE equality-capable yet write no `__hmac`, because the adapter writes it only when the payload carries an `hm` term (gate at helpers.ts:251), which the FFI emits only for the `unique` index. Added __tests__/dynamodb/hmac-domains.test.ts pinning, for every one of the 40 v3 domains, whether it builds the `unique` index that produces `__hmac` — the ground truth the doc rests on. The definitive set is 12 domains (9 `*Eq` + TextOrd + TextOrdOre + TextSearch); running it corrected an off-by-one in the review's stated count (13). Reworded line 39 to match the test and the table. F1 — three encryptQuery snippets (lines 492, 499, 548) read `result.data?.hm` without checking the failure branch; `?.` masks an encryption failure (and a null-plaintext result) as `undefined`, producing a malformed key condition instead of a clear error. All three now check `if (result.failure) throw` first, matching the correct example already in the file at line 318. F2 — sharpened the v2/v3 "no data migration" wording so it can't be misread as "none needed": no infrastructure migration, but no automatic data migration either — items written under one version cannot be decrypted by the other. No changeset needed: the branch's existing dynamodb-eql-v3.md already carries a `stash` patch, which covers the skill; the new test is test-only. --- .../__tests__/dynamodb/hmac-domains.test.ts | 77 +++++++++++++++++++ skills/stash-dynamodb/SKILL.md | 16 ++-- 2 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 packages/stack/__tests__/dynamodb/hmac-domains.test.ts diff --git a/packages/stack/__tests__/dynamodb/hmac-domains.test.ts b/packages/stack/__tests__/dynamodb/hmac-domains.test.ts new file mode 100644 index 00000000..d1108e42 --- /dev/null +++ b/packages/stack/__tests__/dynamodb/hmac-domains.test.ts @@ -0,0 +1,77 @@ +/** + * Ground truth for which EQL v3 domains produce a DynamoDB `__hmac` attribute. + * + * On DynamoDB, `encryptedDynamoDB` writes `__hmac` only when the encrypted + * payload carries an `hm` equality term (the gate is `if (encryptPayload.hm)` in + * `helpers.ts`). The FFI emits `hm` exactly when the column's `build()` config + * carries the `unique` index, minted by `indexesForCapabilities` + * (`eql/v3/columns.ts`): a column is `equality`-capable AND either not ordering + * or text-cast. So `unique`-index presence per domain IS the ground truth for + * `__hmac`, and this test pins it for EVERY `types.*` domain. + * + * It exists because the `skills/stash-dynamodb` doc makes a per-domain claim + * about `__hmac` and nothing else checks it — skills ship to customers and drift + * silently. If a domain's capabilities change, this fails and the doc's domain + * table must be re-checked against it. + * + * Note the subtlety this locks: several `*Ord` domains are `equality`-capable + * (they answer equality via an ordering term in Postgres) yet write NO `__hmac`, + * because equality on DynamoDB needs the deterministic `hm`, which only the + * `unique` index provides. "Equality-capable" is therefore NOT the same set as + * "writes `__hmac`". + */ +import { describe, expect, it } from 'vitest' +import { types } from '@/eql/v3' + +/** + * The domains that mint an `hm` equality term, and so write `__hmac`: + * every `*Eq`, plus the text-cast ordering/search domains (text equality is + * always HMAC-based). Everything else — non-text `*Ord`/`*OrdOre`, the bare + * storage-only domains, `TextMatch`, and `Json` — does not. + */ +const WRITES_HMAC = new Set([ + 'TextEq', + 'IntegerEq', + 'SmallintEq', + 'BigintEq', + 'DateEq', + 'TimestampEq', + 'NumericEq', + 'RealEq', + 'DoubleEq', + 'TextOrd', + 'TextOrdOre', + 'TextSearch', +]) + +/** Does this domain's column config carry the `unique` index (→ `hm` → `__hmac`)? */ +function buildsUniqueIndex(factory: (name: string) => unknown): boolean { + const column = factory('c') as { + build(): { indexes: Record } + } + return 'unique' in column.build().indexes +} + +const allDomains = Object.keys(types) as Array + +describe('EQL v3 domain → __hmac ground truth', () => { + it('covers every domain the skill can reference', () => { + // Guard against a new domain being added without a verdict here. + expect(allDomains.length).toBe(40) + }) + + it.each(allDomains)('%s matches the documented __hmac set', (name) => { + const factory = types[name] as (n: string) => unknown + expect(buildsUniqueIndex(factory)).toBe(WRITES_HMAC.has(name as string)) + }) + + it('is exactly the 12 documented domains — no more, no fewer', () => { + const actual = allDomains + .filter((n) => buildsUniqueIndex(types[n] as (x: string) => unknown)) + .map((n) => n as string) + .sort() + + expect(actual).toEqual([...WRITES_HMAC].sort()) + expect(actual).toHaveLength(12) + }) +}) diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index 2dca84cf..355f16be 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -36,7 +36,7 @@ CipherStash encrypts each attribute into two DynamoDB attributes: | Original Attribute | Stored As | Purpose | |---|---|---| | `email` | `email__source` | Encrypted ciphertext | -| `email` | `email__hmac` | HMAC for equality lookups (only for equality-capable columns) | +| `email` | `email__hmac` | HMAC — written whenever the domain produces an equality term, usable only for equality lookups (see the domain table below) | Non-encrypted attributes pass through unchanged. On decryption, the `__source` and `__hmac` attributes are recombined back into the original attribute name with the plaintext value. @@ -53,7 +53,7 @@ Non-encrypted attributes pass through unchanged. On decryption, the `__source` a | Client | `EncryptionV3({ schemas })` | `Encryption({ schemas })` | | Nested fields | Flat dotted path (`"profile.ssn"`) | Nested group + `encryptedField` | -There is no data migration between them: DynamoDB has no EQL extension to install and no schema to alter. But the two write **different wire formats**, so a table populated under v2 cannot be read back with a v3 schema, or vice versa. Pick one per table and stay on it. +There is no infrastructure migration between them — DynamoDB has no EQL extension to install and no schema to alter — but there is no automatic *data* migration path either. The two write **different wire formats**, so items written under one version cannot be decrypted by the other. Pick one per table and stay on it; to switch an existing table you must re-encrypt every item. **Nested attributes work in both versions**, with different authoring syntax. @@ -489,14 +489,19 @@ const result = await encryptionClient.encryptQuery( "search-value", { table: users, column: users.email } ) -const hmac = result.data?.hm // Use this in DynamoDB key conditions +// encryptQuery returns a Result; check the failure branch before reading `data`. +// `data?.hm` would mask a failure (and a null-plaintext result) as `undefined`, +// producing a malformed key condition rather than a clear error. +if (result.failure) throw new Error(result.failure.message) +const hmac = result.data.hm // Use this in DynamoDB key conditions // EQL v2 — pass queryType explicitly: const v2Result = await encryptionClient.encryptQuery( "search-value", { table: users, column: users.email, queryType: "equality" } ) -const v2Hmac = v2Result.data?.hm +if (v2Result.failure) throw new Error(v2Result.failure.message) +const v2Hmac = v2Result.data.hm ``` > On a `types.TextSearch` column `encryptQuery` returns `hm` alongside ordering @@ -545,7 +550,8 @@ const queryEnc = await encryptionClient.encryptQuery("alice@example.com", { table: users, column: users.email, }) -const hmac = queryEnc.data?.hm +if (queryEnc.failure) throw new Error(queryEnc.failure.message) +const hmac = queryEnc.data.hm const queryResult = await docClient.send(new QueryCommand({ TableName: "Users", From 997a4c2dfaa561110fff9ee05f50f4ff2061b2fb Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 21 Jul 2026 10:43:28 +1000 Subject: [PATCH 12/20] docs(skills): clarify v2/v3 produce the same layout, not the same item MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit flagged 'Both produce the same DynamoDB item' as misleading: v2 and v3 produce the same attribute LAYOUT (identical __source/__hmac keys), but the encrypted contents are version-specific and not cross-decryptable — which the compatibility note earlier in the file already states. Reworded to say layout, not item, and cross-link the incompatibility. --- skills/stash-dynamodb/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index 355f16be..9c2b8614 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -76,7 +76,7 @@ const users = encryptedTable("users", { }) ``` -Both produce the same DynamoDB item, and the v3 form keeps equality queryability on the nested attribute: +Both produce the same DynamoDB attribute *layout* — and the v3 form keeps equality queryability on the nested attribute — but the encrypted contents are version-specific: an item written under one version cannot be decrypted under the other (see the compatibility note above). ```jsonc { "pk": "u#1", From 5e72ae27852cc2d62a6a836d157ed98d6b310f1e Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 21 Jul 2026 11:06:18 +1000 Subject: [PATCH 13/20] docs: fix stale numbers flagged in review (numRuns, test count) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two doc-accuracy findings from CodeRabbit; the other two verified as design-preferences and skipped (see below). - The live property test's docstring said "capped hard at 15 runs" while the code was lowered to `numRuns: 5` last round. Corrected the prose. - The changeset hardcoded "74 tests"; the adapter now has 116 runtime cases. Dropped the number, kept the "first test coverage across v2 and v3" statement. Skipped, with reasons: - Suite-level credential skipIf guard: every sibling live suite under packages/stack/__tests__/ runs beforeAll unconditionally and throws without CS_* creds — that is the repo convention (the throw-not-skip comment in vitest.config.ts). A guard here would diverge, not conform. No requireIntegrationEnv helper is used as a gate by any live suite. - Emit DB column name instead of property name as the stored attribute: not a bug. The DynamoDB attribute name is a free internal choice (no external DDL, unlike Postgres). The current property-name behaviour round-trips and its __hmac matches the query term (verified live, locked by the "property differs from its DB name" tests). CodeRabbit's write-only change would break the round trip — the read path matches by property name via resolveEncryptColumnMap, so a DB-name attribute would miss `matchColumn` and pass through undecrypted. --- .changeset/dynamodb-eql-v3.md | 4 ++-- .../stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md index ee5ba240..1f73b2b3 100644 --- a/.changeset/dynamodb-eql-v3.md +++ b/.changeset/dynamodb-eql-v3.md @@ -36,8 +36,8 @@ Notes on capability: - Audit metadata on `decryptModel` / `bulkDecryptModels` requires the nominal client; the `EncryptionV3` client has no audit surface on decrypt. -The DynamoDB adapter also gains its first test coverage — 74 tests across the -v2 and v3 paths, where it previously had none. +The DynamoDB adapter also gains its first test coverage — across the v2 and v3 +paths, where it previously had none. The v3 overloads are strongly typed. `encryptModel` / `bulkEncryptModels` check the input model against the table's column domains, and return the DynamoDB diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index 9d3d7e10..bbdf62f2 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -197,7 +197,7 @@ describe('round trips with a v3 table', () => { * The one LIVE property. Its pure siblings (`properties.test.ts`) cover the * attribute mapping in both directions with a fake ciphertext; this closes the * loop over the real thing — every run is a ZeroKMS encrypt + decrypt, so it - * is capped hard at 15 runs. It is the only check that the split/rebuild + * is capped hard at 5 runs. It is the only check that the split/rebuild * survives contact with actual ciphertext across all five domains at once. */ it('property: round-trips arbitrary multi-domain items through ZeroKMS', async () => { From 5b26ad3c70149d04d4130ee40fbd776131a69d5e Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 21 Jul 2026 17:14:02 +1000 Subject: [PATCH 14/20] fix(stack): address DynamoDB v3 review findings with regression tests Implements the review fixes from PR #725 (coderdan / coderabbit), each with a failing test written first (behavior changes) or mutation-proven coverage (already-correct paths). Behavior: - deepClone: non-cloneable input now falls back to a fresh per-key clone, so the "encryption never mutates a caller's object" guarantee holds even when structuredClone throws (function/symbol-valued properties). - Read path logs a debug diagnostic when a `*__source` attribute names no declared column, instead of silently returning raw ciphertext. - resolveDecryptResult surfaces a malformed client result as a failure rather than a silent `undefined` success. - encryptedDynamoDB throws a clear, table-named error when a v3 table is used with a client that never registered it (was an opaque FFI failure later). Types/refactor: - Per-method casts on the operations object so drift from the overloaded public type is caught by tsc (was one broad `as`). - Single `hasBuildColumnKeyMap` predicate in the leaf `types.ts`; the four hand-written v3-marker checks now route through it (wasm-inline stays native-free). - Remove dead `isNested` param; convert O(N^2) reduce accumulators to in-place. Tests: v3 bulk-renamed regression, v3 bulk error paths, empty-batch + null column, read-context-once invariant, audit-metadata log, deepClone fallback, type-level HasSearchTerm/DecryptedAttributes coverage, and a suite-level credential guard (describe.skipIf) so the live v3 suite skips without creds. Refs #725 --- .changeset/dynamodb-eql-v3.md | 14 + .../dynamodb/client-compat.test-d.ts | 71 +++ .../dynamodb/construct-guard.test.ts | 126 +++++ .../dynamodb/encrypted-dynamodb-v3.test.ts | 492 +++++++++++------- .../__tests__/dynamodb/helpers-v3.test.ts | 75 ++- .../dynamodb/resolve-decrypt.test.ts | 44 +- packages/stack/src/dynamodb/helpers.ts | 66 ++- packages/stack/src/dynamodb/index.ts | 164 ++++-- packages/stack/src/encryption/index.ts | 5 +- packages/stack/src/types.ts | 18 + packages/stack/src/wasm-inline.ts | 5 +- skills/stash-dynamodb/SKILL.md | 12 +- 12 files changed, 836 insertions(+), 256 deletions(-) create mode 100644 packages/stack/__tests__/dynamodb/construct-guard.test.ts diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md index 1f73b2b3..d5eec1ef 100644 --- a/.changeset/dynamodb-eql-v3.md +++ b/.changeset/dynamodb-eql-v3.md @@ -39,6 +39,20 @@ Notes on capability: The DynamoDB adapter also gains its first test coverage — across the v2 and v3 paths, where it previously had none. +Robustness, from review: + +- Passing a v3 table to a client that never registered it (one built for a + different schema set, so it is not in v3 mode for that table) now throws a + clear, actionable error naming the table, instead of failing opaquely deep in + the FFI. +- A malformed decrypt result from a non-conforming client is surfaced as a + failure rather than resolving as a silent `undefined` success. +- Reading back a `__source` attribute that matches no declared column now + logs a debug diagnostic instead of silently returning the raw ciphertext. +- Caller input that cannot be structurally cloned no longer reaches the FFI by + reference — the "encryption never mutates a caller's object" guarantee holds + on that path too. + The v3 overloads are strongly typed. `encryptModel` / `bulkEncryptModels` check the input model against the table's column domains, and return the DynamoDB attribute map that is actually written — the new exported `EncryptedAttributes` diff --git a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts index ae879dce..ea032ae9 100644 --- a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts +++ b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts @@ -30,6 +30,16 @@ const usersV2 = encryptedTableV2('users_v2', { email: encryptedColumn('email').equality(), }) +// Exercises the INNERMOST `true` arm of `HasSearchTerm` (types.ts): a text +// domain that is BOTH equality- and order/range-capable — text equality is +// always HMAC-based, so it mints `__hmac`. `usersV3.age` (IntegerOrd) reaches +// the same conditional but over `number`, taking the `false` arm; without a +// text-ordering column here that `true` arm had no type-level witness. +const searchV3 = encryptedTableV3('search_v3', { + title: types.TextOrd('title'), // equality + orderAndRange over string → __source + __hmac + bio: types.TextMatch('bio'), // free-text only, no equality → __source, NO __hmac +}) + type V3Model = { pk: string; email?: string; age?: number; role?: string } // The two client shapes. `EncryptionV3` returns a `TypedEncryptionClient` @@ -210,6 +220,67 @@ describe('the v3 overload types the DynamoDB storage split', () => { }) }) +describe('a text-ordering domain reaches the HasSearchTerm `true` arm', () => { + it('mints __hmac for a text equality+ordering column, but not for free-text-only', async () => { + const result = await dynamo.encryptModel( + { pk: 'a', title: 'Hello', bio: 'about me' }, + searchV3, + ) + if (result.failure) return + + // TextOrd is equality- AND order/range-capable over `string`: the innermost + // `[PlaintextForColumn] extends [string] ? true : false` resolves `true`, + // so the queryable HMAC term is present. This is the arm no prior column + // instantiated — `usersV3.age` (IntegerOrd) takes the `false` (number) arm. + expectTypeOf(result.data.title__source).toEqualTypeOf() + expectTypeOf(result.data.title__hmac).toEqualTypeOf() + + // A free-text-only domain (`TextMatch`) is NOT equality-capable, so it never + // reaches the ordering/text arms and writes no HMAC term — the distinct, + // adjacent behaviour that makes the `true` arm meaningful. + expectTypeOf(result.data.bio__source).toEqualTypeOf() + expectTypeOf(result.data).not.toHaveProperty('bio__hmac') + }) +}) + +describe('decrypt passes through suffixed keys whose base names no column', () => { + it('folds a declared column but leaves unrelated __hmac / __source keys intact', async () => { + // `email` IS a declared column; `legit` and `img` are NOT. A stored item can + // legitimately carry a customer attribute that merely ends in `__hmac` + // (an app-level signature) or `__source` (a renamed/foreign column) — the + // runtime read path preserves both, and these two `DecryptedAttributes` arms + // type that: a suffixed key whose base names no column is returned untouched. + const result = await dynamo.decryptModel( + { + pk: 'a', + email__source: 'ct', + email__hmac: 'hm', + legit__hmac: 'sig', + img__source: 'raw', + }, + usersV3, + ) + if (result.failure) return + + // Declared column: `email__source` folds to `email`, `email__hmac` (its base + // IS a column) is dropped as a query term. + expectTypeOf(result.data.email).toEqualTypeOf() + expectTypeOf(result.data).not.toHaveProperty('email__source') + expectTypeOf(result.data).not.toHaveProperty('email__hmac') + + // Non-column suffixed keys: neither folded nor dropped — the two "names no + // column" arms. `legit__hmac`'s base is not a column, so unlike `email__hmac` + // it survives; `img__source`'s base is not a column, so unlike `email__source` + // it is NOT folded to `img`. + expectTypeOf(result.data.legit__hmac).toEqualTypeOf() + expectTypeOf(result.data.img__source).toEqualTypeOf() + expectTypeOf(result.data).not.toHaveProperty('img') + + // Ordinary passthrough attribute, for contrast. + expectTypeOf(result.data.pk).toEqualTypeOf() + }) +}) + describe('the v2 overload still returns the input model', () => { it('keeps an existing v2 caller compiling unchanged', async () => { const result = await dynamo.encryptModel<{ pk: string; email?: string }>( diff --git a/packages/stack/__tests__/dynamodb/construct-guard.test.ts b/packages/stack/__tests__/dynamodb/construct-guard.test.ts new file mode 100644 index 00000000..65ba86f2 --- /dev/null +++ b/packages/stack/__tests__/dynamodb/construct-guard.test.ts @@ -0,0 +1,126 @@ +/** + * Construction-time client/table EQL-version guard (#725, thread 13). + * + * A v3 table handed to a client that was NOT built in EQL v3 mode for it (a + * v2-mode client, or one initialised for a different schema set) otherwise + * fails only much later, deep inside the FFI, with an opaque deserialization + * error. `encryptedDynamoDB`'s operation methods guard against that mismatch + * the moment a table is supplied — the earliest point at which BOTH the client + * and the table version are known — and throw a clear, actionable Error. + * + * This suite is pure: it drives the guard with stub clients whose + * `getEncryptConfig()` reports which tables the client knows about, so it needs + * neither credentials nor a live FFI round-trip. The stub operation methods + * throw if ever reached, proving the guard fires BEFORE any client call. + */ +import { describe, expect, it } from 'vitest' +import { encryptedDynamoDB } from '@/dynamodb' +import { encryptedTable as encryptedTableV3, types } from '@/eql/v3' +import { encryptedColumn, encryptedTable as encryptedTableV2 } from '@/schema' + +const usersV3 = encryptedTableV3('users_v3', { + email: types.TextEq('email'), +}) + +const usersV2 = encryptedTableV2('users_v2', { + email: encryptedColumn('email').equality(), +}) + +/** + * A stub client that reports the given table names in its encrypt config. The + * operation methods throw if invoked, so any test that reaches them (rather than + * the synchronous guard) fails loudly. + */ +function stubClient(knownTables: string[]) { + const tables = Object.fromEntries(knownTables.map((t) => [t, {}])) + const unreached = () => { + throw new Error('FFI operation reached — guard did not fire first') + } + return { + getEncryptConfig: () => ({ v: 1, tables }), + encryptModel: unreached, + bulkEncryptModels: unreached, + decryptModel: unreached, + bulkDecryptModels: unreached, + // biome-ignore lint/suspicious/noExplicitAny: test stub, .test.ts is not typechecked + } as any +} + +describe('encryptedDynamoDB client/table version guard', () => { + it('throws a clear version-mismatch error for a v3 table on a client that does not know it', () => { + // A client built for other (e.g. v2) schemas: `users_v3` is absent. + const dynamo = encryptedDynamoDB({ encryptionClient: stubClient([]) }) + + expect(() => + dynamo.encryptModel({ pk: 'a', email: 'a@b.com' }, usersV3), + ).toThrowError(/version mismatch/i) + expect(() => + dynamo.encryptModel({ pk: 'a', email: 'a@b.com' }, usersV3), + ).toThrowError(/v3/) + }) + + it('names the offending table and how to fix it', () => { + const dynamo = encryptedDynamoDB({ encryptionClient: stubClient([]) }) + let message = '' + try { + dynamo.encryptModel({ pk: 'a', email: 'a@b.com' }, usersV3) + } catch (e) { + message = (e as Error).message + } + expect(message).toContain('users_v3') + expect(message).toMatch(/EncryptionV3|eqlVersion/) + }) + + it('guards every operation method, not just encryptModel', () => { + const dynamo = encryptedDynamoDB({ encryptionClient: stubClient([]) }) + expect(() => + dynamo.bulkEncryptModels([{ pk: 'a', email: 'a@b.com' }], usersV3), + ).toThrowError(/version mismatch/i) + expect(() => + dynamo.decryptModel({ pk: 'a', email__source: 'ct' }, usersV3), + ).toThrowError(/version mismatch/i) + expect(() => + dynamo.bulkDecryptModels([{ pk: 'a', email__source: 'ct' }], usersV3), + ).toThrowError(/version mismatch/i) + }) + + it('does NOT throw when a v3 table is registered with the client (matching versions)', () => { + const dynamo = encryptedDynamoDB({ + encryptionClient: stubClient(['users_v3']), + }) + // Constructing the operation must not throw; the stub op methods are never + // awaited, so the "unreached" guard inside them is not tripped. + expect(() => + dynamo.encryptModel({ pk: 'a', email: 'a@b.com' }, usersV3), + ).not.toThrow() + }) + + it('does NOT throw for a v2 table on a v2-mode client (matching versions)', () => { + const dynamo = encryptedDynamoDB({ + encryptionClient: stubClient(['users_v2']), + }) + expect(() => + dynamo.encryptModel({ pk: 'a', email: 'a@b.com' }, usersV2), + ).not.toThrow() + }) + + it('does NOT throw a v3 table on a client whose config cannot be read (cannot determine version)', () => { + // A client shape with no getEncryptConfig — the guard cannot prove a + // mismatch, so it must not false-positive. + const unreached = () => { + throw new Error('FFI operation reached — guard did not fire first') + } + const dynamo = encryptedDynamoDB({ + encryptionClient: { + encryptModel: unreached, + bulkEncryptModels: unreached, + decryptModel: unreached, + bulkDecryptModels: unreached, + // biome-ignore lint/suspicious/noExplicitAny: test stub + } as any, + }) + expect(() => + dynamo.encryptModel({ pk: 'a', email: 'a@b.com' }, usersV3), + ).not.toThrow() + }) +}) diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index bbdf62f2..04c73cc3 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -26,6 +26,19 @@ import type { JsonValue } from '@/eql/v3' import { encryptedTable, types } from '@/eql/v3' import { Encryption } from '@/index' +/** + * This suite talks to live ZeroKMS. Skip (rather than fail) when the CipherStash + * credentials are absent, so a local run without `.env` is green — matching the + * four variables `requireIntegrationEnv(['cipherstash'])` checks. With creds + * present the suite runs normally. + */ +const hasCipherStashCreds = [ + 'CS_WORKSPACE_CRN', + 'CS_CLIENT_ID', + 'CS_CLIENT_KEY', + 'CS_CLIENT_ACCESS_KEY', +].every((name) => Boolean(process.env[name])) + const users = encryptedTable('users_v3_dynamo', { email: types.TextEq('email'), name: types.Text('name'), @@ -54,6 +67,10 @@ let nominalClient: EncryptionClient let nominalDynamo: EncryptedDynamoDBInstance beforeAll(async () => { + // Nothing to set up when the suite is skipped for lack of credentials; the + // client constructors below would otherwise fail to authenticate. + if (!hasCipherStashCreds) return + const typedClient = await EncryptionV3({ schemas: [users] }) typedDynamo = encryptedDynamoDB({ encryptionClient: typedClient }) @@ -64,7 +81,7 @@ beforeAll(async () => { nominalDynamo = encryptedDynamoDB({ encryptionClient: nominalClient }) }) -describe('encryptModel with a v3 table', () => { +describe.skipIf(!hasCipherStashCreds)('encryptModel with a v3 table', () => { it('splits an equality domain into __source and __hmac', async () => { const result = await typedDynamo.encryptModel( { pk: 'user#1', email: 'alice@example.com', role: 'admin' }, @@ -141,9 +158,37 @@ describe('encryptModel with a v3 table', () => { expect(Array.isArray(result.data.meta__source)).toBe(true) expect(result.data.meta__source.length).toBeGreaterThan(0) }) + + it('passes a null column value through without splitting it', async () => { + // v3 reaches the null check through a different code path than v2: + // `isStoredEqlPayload` runs before the `k`/`c` gates, so a null must be + // recognised as a non-payload and passed through verbatim — no + // `email__source`, no `email__hmac` — with siblings intact. + const result = await typedDynamo.encryptModel( + { pk: 'user#null', email: null, role: 'admin' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(result.data.email).toBeNull() + expect(result.data).not.toHaveProperty('email__source') + expect(result.data).not.toHaveProperty('email__hmac') + expect(result.data.pk).toBe('user#null') + expect(result.data.role).toBe('admin') + + const decrypted = await typedDynamo.decryptModel(result.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual({ + pk: 'user#null', + email: null, + role: 'admin', + }) + }) }) -describe('round trips with a v3 table', () => { +describe.skipIf(!hasCipherStashCreds)('round trips with a v3 table', () => { it('round-trips every domain back to the original item', async () => { const original: User = { pk: 'user#6', @@ -239,7 +284,7 @@ describe('round trips with a v3 table', () => { }, 120000) }) -describe('bulk operations with a v3 table', () => { +describe.skipIf(!hasCipherStashCreds)('bulk operations with a v3 table', () => { it('encrypts and decrypts a batch', async () => { const items: User[] = [ { pk: 'user#9', email: 'a@example.com', name: 'A' }, @@ -278,221 +323,278 @@ describe('bulk operations with a v3 table', () => { expect(decrypted.data).toEqual(items) }) -}) - -describe('nested attributes with a v3 table', () => { - // EQL v3 has no nested-object *authoring* form — a nested group is a compile - // error. But a dotted column path is a supported column name, and the model - // walk in `encryption/helpers/model-helpers.ts` is shared with v2 and matches - // on dotted paths. So nested DynamoDB attributes work in v3 today, declared - // flat. This matters for DynamoDB specifically, where items are natively - // nested documents. - const nested = encryptedTable('users_v3_nested', { - 'profile.ssn': types.TextEq('profile.ssn'), - 'profile.note': types.Text('profile.note'), - }) - - type NestedUser = { - pk: string - profile?: { ssn?: string; note?: string; city?: string } - } - let nestedDynamo: EncryptedDynamoDBInstance - let nestedClient: EncryptionClient + it('accepts an empty batch', async () => { + const encrypted = await typedDynamo.bulkEncryptModels([], users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - beforeAll(async () => { - nestedClient = await Encryption({ - schemas: [nested] as never, - config: { eqlVersion: 3 }, - }) - nestedDynamo = encryptedDynamoDB({ encryptionClient: nestedClient }) + expect(encrypted.data).toEqual([]) }) +}) - it('splits nested attributes in place, leaving plaintext siblings alone', async () => { - const result = await nestedDynamo.encryptModel( - { - pk: 'u#1', - profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, - }, - nested, - ) - - if (result.failure) throw new Error(result.failure.message) - - const profile = result.data.profile as Record - expect(Object.keys(profile).sort()).toEqual([ - 'city', - 'note__source', - 'ssn__hmac', - 'ssn__source', - ]) - expect(profile.city).toBe('Sydney') - }) +describe.skipIf(!hasCipherStashCreds)( + 'nested attributes with a v3 table', + () => { + // EQL v3 has no nested-object *authoring* form — a nested group is a compile + // error. But a dotted column path is a supported column name, and the model + // walk in `encryption/helpers/model-helpers.ts` is shared with v2 and matches + // on dotted paths. So nested DynamoDB attributes work in v3 today, declared + // flat. This matters for DynamoDB specifically, where items are natively + // nested documents. + const nested = encryptedTable('users_v3_nested', { + 'profile.ssn': types.TextEq('profile.ssn'), + 'profile.note': types.Text('profile.note'), + }) - it('round-trips a nested item exactly', async () => { - const original: NestedUser = { - pk: 'u#2', - profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, + type NestedUser = { + pk: string + profile?: { ssn?: string; note?: string; city?: string } } - const encrypted = await nestedDynamo.encryptModel(original, nested) - if (encrypted.failure) throw new Error(encrypted.failure.message) - - const decrypted = await nestedDynamo.decryptModel(encrypted.data, nested) - if (decrypted.failure) throw new Error(decrypted.failure.message) - - expect(decrypted.data).toEqual(original) - }) + let nestedDynamo: EncryptedDynamoDBInstance + let nestedClient: EncryptionClient - it('rebuilds the nested identifier from its registered dotted name', async () => { - const encrypted = await nestedDynamo.encryptModel( - { pk: 'u#3', profile: { ssn: '123-45-6789' } }, - nested, - ) - if (encrypted.failure) throw new Error(encrypted.failure.message) + beforeAll(async () => { + nestedClient = await Encryption({ + schemas: [nested] as never, + config: { eqlVersion: 3 }, + }) + nestedDynamo = encryptedDynamoDB({ encryptionClient: nestedClient }) + }) - const rebuilt = toItemWithEqlPayloads(encrypted.data, nested) - const payload = (rebuilt.profile as Record) - .ssn + it('splits nested attributes in place, leaving plaintext siblings alone', async () => { + const result = await nestedDynamo.encryptModel( + { + pk: 'u#1', + profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, + }, + nested, + ) + + if (result.failure) throw new Error(result.failure.message) + + const profile = result.data.profile as Record + expect(Object.keys(profile).sort()).toEqual([ + 'city', + 'note__source', + 'ssn__hmac', + 'ssn__source', + ]) + expect(profile.city).toBe('Sydney') + }) - expect(payload.i.c).toBe('profile.ssn') - }) + it('round-trips a nested item exactly', async () => { + const original: NestedUser = { + pk: 'u#2', + profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, + } - it('makes a nested equality attribute queryable by __hmac', async () => { - const ssn = '999-88-7777' + const encrypted = await nestedDynamo.encryptModel(original, nested) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const stored = await nestedDynamo.encryptModel( - { pk: 'u#4', profile: { ssn } }, - nested, - ) - if (stored.failure) throw new Error(stored.failure.message) + const decrypted = await nestedDynamo.decryptModel(encrypted.data, nested) + if (decrypted.failure) throw new Error(decrypted.failure.message) - const term = await nestedClient.encryptQuery(ssn, { - table: nested as never, - column: nested['profile.ssn'] as never, + expect(decrypted.data).toEqual(original) }) - if (term.failure) throw new Error(term.failure.message) - const profile = stored.data.profile as Record - // A key condition on `profile.ssn__hmac` matches what was written. - expect((term.data as { hm: string }).hm).toBe(profile.ssn__hmac) - }) + it('rebuilds the nested identifier from its registered dotted name', async () => { + const encrypted = await nestedDynamo.encryptModel( + { pk: 'u#3', profile: { ssn: '123-45-6789' } }, + nested, + ) + if (encrypted.failure) throw new Error(encrypted.failure.message) - it('bulk round-trips nested items', async () => { - const items: NestedUser[] = [ - { pk: 'u#5', profile: { ssn: 'a', city: 'Sydney' } }, - { pk: 'u#6', profile: { note: 'b' } }, - ] + const rebuilt = toItemWithEqlPayloads(encrypted.data, nested) + const payload = (rebuilt.profile as Record) + .ssn - const encrypted = await nestedDynamo.bulkEncryptModels(items, nested) - if (encrypted.failure) throw new Error(encrypted.failure.message) + expect(payload.i.c).toBe('profile.ssn') + }) - const decrypted = await nestedDynamo.bulkDecryptModels( - encrypted.data, - nested, - ) - if (decrypted.failure) throw new Error(decrypted.failure.message) + it('makes a nested equality attribute queryable by __hmac', async () => { + const ssn = '999-88-7777' + + const stored = await nestedDynamo.encryptModel( + { pk: 'u#4', profile: { ssn } }, + nested, + ) + if (stored.failure) throw new Error(stored.failure.message) + + const term = await nestedClient.encryptQuery(ssn, { + table: nested as never, + column: nested['profile.ssn'] as never, + }) + if (term.failure) throw new Error(term.failure.message) + + const profile = stored.data.profile as Record + // The query term matches the stored nested `profile.ssn__hmac` — usable in a + // FilterExpression (a nested attribute can't back a DynamoDB key condition). + expect((term.data as { hm: string }).hm).toBe(profile.ssn__hmac) + }) - expect(decrypted.data).toEqual(items) - }) -}) + it('bulk round-trips nested items', async () => { + const items: NestedUser[] = [ + { pk: 'u#5', profile: { ssn: 'a', city: 'Sydney' } }, + { pk: 'u#6', profile: { note: 'b' } }, + ] -describe('a v3 column whose property differs from its DB name', () => { - // Regression: `encryptedAttrs` was derived from `build().columns`, which for - // v3 is keyed by DB name, while the encrypted model is keyed by property - // name. They never matched, so the attribute was never split — no __source, - // no __hmac, and the payload's `i` block was mangled to `i__source`. Decrypt - // still round-tripped, so nothing surfaced it; only a key condition that - // silently matched nothing would have. - const renamed = encryptedTable('users_v3_renamed', { - emailAddress: types.TextEq('email_address'), - }) + const encrypted = await nestedDynamo.bulkEncryptModels(items, nested) + if (encrypted.failure) throw new Error(encrypted.failure.message) - let renamedDynamo: EncryptedDynamoDBInstance - let renamedClient: EncryptionClient + const decrypted = await nestedDynamo.bulkDecryptModels( + encrypted.data, + nested, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) - beforeAll(async () => { - renamedClient = await Encryption({ - schemas: [renamed] as never, - config: { eqlVersion: 3 }, + expect(decrypted.data).toEqual(items) + }) + }, +) + +describe.skipIf(!hasCipherStashCreds)( + 'a v3 column whose property differs from its DB name', + () => { + // Regression: `encryptedAttrs` was derived from `build().columns`, which for + // v3 is keyed by DB name, while the encrypted model is keyed by property + // name. They never matched, so the attribute was never split — no __source, + // no __hmac, and the payload's `i` block was mangled to `i__source`. Decrypt + // still round-tripped, so nothing surfaced it; only a key condition that + // silently matched nothing would have. + const renamed = encryptedTable('users_v3_renamed', { + emailAddress: types.TextEq('email_address'), }) - renamedDynamo = encryptedDynamoDB({ encryptionClient: renamedClient }) - }) - - it('splits the attribute under its property name, with an __hmac', async () => { - const result = await renamedDynamo.encryptModel( - { id: '1', emailAddress: 'a@b.com' }, - renamed, - ) - if (result.failure) throw new Error(result.failure.message) - - expect(Object.keys(result.data).sort()).toEqual([ - 'emailAddress__hmac', - 'emailAddress__source', - 'id', - ]) - // The payload's identifier block must never leak into an attribute. - expect(JSON.stringify(result.data)).not.toContain('i__source') - }) - - it('round-trips, and the query term matches the stored __hmac', async () => { - const original = { id: '2', emailAddress: 'c@d.com' } - const encrypted = await renamedDynamo.encryptModel(original, renamed) - if (encrypted.failure) throw new Error(encrypted.failure.message) + let renamedDynamo: EncryptedDynamoDBInstance + let renamedClient: EncryptionClient - const decrypted = await renamedDynamo.decryptModel(encrypted.data, renamed) - if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(original) + beforeAll(async () => { + renamedClient = await Encryption({ + schemas: [renamed] as never, + config: { eqlVersion: 3 }, + }) + renamedDynamo = encryptedDynamoDB({ encryptionClient: renamedClient }) + }) - const term = await renamedClient.encryptQuery('c@d.com', { - table: renamed as never, - column: renamed.emailAddress as never, + it('splits the attribute under its property name, with an __hmac', async () => { + const result = await renamedDynamo.encryptModel( + { id: '1', emailAddress: 'a@b.com' }, + renamed, + ) + if (result.failure) throw new Error(result.failure.message) + + expect(Object.keys(result.data).sort()).toEqual([ + 'emailAddress__hmac', + 'emailAddress__source', + 'id', + ]) + // The payload's identifier block must never leak into an attribute. + expect(JSON.stringify(result.data)).not.toContain('i__source') }) - if (term.failure) throw new Error(term.failure.message) - expect((term.data as { hm: string }).hm).toBe( - (encrypted.data as Record).emailAddress__hmac, - ) - }) -}) + it('round-trips, and the query term matches the stored __hmac', async () => { + const original = { id: '2', emailAddress: 'c@d.com' } -describe('the __hmac key-condition path with a v3 table', () => { - it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { - const email = 'heidi@example.com' + const encrypted = await renamedDynamo.encryptModel(original, renamed) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const stored = await typedDynamo.encryptModel( - { pk: 'user#13', email }, - users, - ) - if (stored.failure) throw new Error(stored.failure.message) + const decrypted = await renamedDynamo.decryptModel( + encrypted.data, + renamed, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data).toEqual(original) - const term = await nominalClient.encryptQuery(email, { - table: users as never, - column: users.email as never, + const term = await renamedClient.encryptQuery('c@d.com', { + table: renamed as never, + column: renamed.emailAddress as never, + }) + if (term.failure) throw new Error(term.failure.message) + + expect((term.data as { hm: string }).hm).toBe( + (encrypted.data as Record).emailAddress__hmac, + ) }) - if (term.failure) throw new Error(term.failure.message) - // `encryptQuery` on a v3 equality domain mints the bare term — `{v, i, hm}` - // with no ciphertext — so the key condition consumes `hm` directly. - expect(term.data).not.toHaveProperty('c') - expect((term.data as { hm: string }).hm).toBe(stored.data.email__hmac) - }) + it('splits each item under its property name on the bulk path, and round-trips', async () => { + // Guards the SAME regression on `bulkEncryptModels`: its `encryptedAttrs` + // must come from `resolveEncryptColumnMap` (property names), not + // `build().columns` (DB names). With the DB-name form, every item would be + // stored under `email_address__*` — or not split at all — and no key + // condition on `emailAddress__hmac` would ever match. + const items = [ + { id: 'b1', emailAddress: 'e@f.com' }, + { id: 'b2', emailAddress: 'g@h.com' }, + ] + + const encrypted = await renamedDynamo.bulkEncryptModels(items, renamed) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + expect(encrypted.data).toHaveLength(2) + for (const item of encrypted.data) { + const stored = item as Record + expect(Object.keys(stored).sort()).toEqual([ + 'emailAddress__hmac', + 'emailAddress__source', + 'id', + ]) + // The DB-name form would have produced these instead of the property-name + // attributes above. + expect(stored).not.toHaveProperty('email_address__source') + expect(stored).not.toHaveProperty('email_address__hmac') + } + + const decrypted = await renamedDynamo.bulkDecryptModels( + encrypted.data, + renamed, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(items) + }) + }, +) + +describe.skipIf(!hasCipherStashCreds)( + 'the __hmac key-condition path with a v3 table', + () => { + it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { + const email = 'heidi@example.com' + + const stored = await typedDynamo.encryptModel( + { pk: 'user#13', email }, + users, + ) + if (stored.failure) throw new Error(stored.failure.message) + + const term = await nominalClient.encryptQuery(email, { + table: users as never, + column: users.email as never, + }) + if (term.failure) throw new Error(term.failure.message) + + // `encryptQuery` on a v3 equality domain mints the bare term — `{v, i, hm}` + // with no ciphertext — so the key condition consumes `hm` directly. + expect(term.data).not.toHaveProperty('c') + expect((term.data as { hm: string }).hm).toBe(stored.data.email__hmac) + }) - it('is deterministic across separate encryptions of the same value', async () => { - const email = 'ivan@example.com' + it('is deterministic across separate encryptions of the same value', async () => { + const email = 'ivan@example.com' - const first = await typedDynamo.encryptModel({ pk: 'a', email }, users) - const second = await typedDynamo.encryptModel({ pk: 'b', email }, users) - if (first.failure) throw new Error(first.failure.message) - if (second.failure) throw new Error(second.failure.message) + const first = await typedDynamo.encryptModel({ pk: 'a', email }, users) + const second = await typedDynamo.encryptModel({ pk: 'b', email }, users) + if (first.failure) throw new Error(first.failure.message) + if (second.failure) throw new Error(second.failure.message) - expect(first.data.email__hmac).toBe(second.data.email__hmac) - expect(first.data.email__source).not.toBe(second.data.email__source) - }) -}) + expect(first.data.email__hmac).toBe(second.data.email__hmac) + expect(first.data.email__source).not.toBe(second.data.email__source) + }) + }, +) -describe('audit metadata with a v3 table', () => { +describe.skipIf(!hasCipherStashCreds)('audit metadata with a v3 table', () => { const metadata = { sub: 'user-id-123', action: 'v3-port' } it('is carried on every operation of the nominal client', async () => { @@ -531,7 +633,7 @@ describe('audit metadata with a v3 table', () => { }) }) -describe('error handling with a v3 table', () => { +describe.skipIf(!hasCipherStashCreds)('error handling with a v3 table', () => { const unknown = encryptedTable('users_v3_dynamo', { nope: types.TextEq('nonexistent_column'), }) @@ -559,6 +661,32 @@ describe('error handling with a v3 table', () => { expect(result.failure?.details).toEqual({ context: 'decryptModel' }) }) + it('surfaces the FFI error code on the bulk encrypt path', async () => { + const result = await typedDynamo.bulkEncryptModels( + [{ nope: 'value' }], + unknown, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) + + it('surfaces the FFI ciphertext error code on the bulk decrypt path', async () => { + // The v3 bulk decrypt failure path (resolveDecryptResult + + // throwPreservingCode against a typed client) is otherwise never exercised + // end-to-end. Like the single-item v3 path, a malformed __source is rejected + // as "Invalid EQL ciphertext" with a real FFI code — not the v2 fallback. + const result = await typedDynamo.bulkDecryptModels( + [{ email__source: 'not-a-ciphertext' }], + users, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.name).toBe('EncryptedDynamoDBError') + expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') + expect(result.failure?.details).toEqual({ context: 'bulkDecryptModels' }) + }) + it('routes v3 failures to the configured errorHandler', async () => { const seen: string[] = [] const instrumented = encryptedDynamoDB({ diff --git a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts index 144a1892..3ce87b9a 100644 --- a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts @@ -13,8 +13,9 @@ * - a v3 JSON document keeps `k: 'sv'`, which is mandatory — deserialization * fails with "missing field `k`" without it. */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { + buildReadContext, deepClone, isV3Table, toEncryptedDynamoItem, @@ -26,6 +27,7 @@ import { encryptedField, encryptedTable as encryptedTableV2, } from '@/schema' +import { logger } from '@/utils/logger' const users = encryptedTable('users', { email: types.TextEq('email'), @@ -229,6 +231,25 @@ describe('regressions found in review', () => { expect(toItemWithEqlPayloads(item, users)).toEqual(item) }) + it('logs a debug when a *__source attribute names no declared column', () => { + // A schema rename leaves the stored `__source` orphaned; it reads back + // as raw base64 with no error. Make the silent drop observable. + const spy = vi.spyOn(logger, 'debug').mockImplementation(() => {}) + + try { + toItemWithEqlPayloads({ unknown__source: 'CT' }, users) + + expect(spy).toHaveBeenCalledWith( + expect.stringContaining('unknown__source'), + ) + expect(spy).toHaveBeenCalledWith( + expect.stringContaining('no declared column'), + ) + } finally { + spy.mockRestore() + } + }) + it('does not rebuild a nested __source whose leaf collides with a top-level column', () => { // `note` is a TOP-LEVEL column; there is no `profile.note`. A v3 table // registers full dotted paths, so a nested `note__source` must NOT match @@ -299,6 +320,31 @@ describe('regressions found in review', () => { }) }) +describe('read context is resolved once per batch', () => { + it('does not call table.build() per item when a context is passed', () => { + // `buildReadContext` resolves the row-invariant facts once; the 3-arg form + // of `toItemWithEqlPayloads` reuses them. Dropping the 3rd arg would rebuild + // (and re-`build()`) per item — the invariant this pins. + const buildSpy = vi.spyOn(users, 'build') + + const context = buildReadContext(users) + buildSpy.mockClear() + + const items = [ + { email__source: 'a' }, + { email__source: 'b' }, + { email__source: 'c' }, + ] + for (const item of items) { + toItemWithEqlPayloads(item, users, context) + } + + expect(buildSpy).not.toHaveBeenCalled() + + buildSpy.mockRestore() + }) +}) + describe('deepClone preserves structured values', () => { it('preserves Date instances — the whole Timestamp domain family depends on it', () => { const at = new Date('2020-01-02T03:04:05.000Z') @@ -319,4 +365,31 @@ describe('deepClone preserves structured values', () => { expect(() => deepClone(o)).not.toThrow() }) + + it('never hands back the caller original when structuredClone throws', () => { + // A function-valued property makes `structuredClone` throw. The catch must + // still produce a NEW object — returning the original voids the docblock + // guarantee that encryption never mutates a caller's object. + const input = { pk: 'u#1', onSave: () => {} } + + const cloned = deepClone(input) + + expect(cloned).not.toBe(input) + expect(cloned.pk).toBe('u#1') + }) + + it('clones a class instance into a plain object, dropping the prototype', () => { + class Model { + pk = 'u#1' + // An own function property forces the structuredClone fallback. + onSave = () => {} + } + const input = new Model() + + const cloned = deepClone(input) + + expect(cloned).not.toBe(input) + expect(cloned).not.toBeInstanceOf(Model) + expect(cloned.pk).toBe('u#1') + }) }) diff --git a/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts b/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts index 27dab582..b1f74e09 100644 --- a/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts +++ b/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts @@ -5,8 +5,9 @@ * reachable through a live ZeroKMS decrypt; these move that assurance onto the * pure CI lane. No credentials, no network. */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { resolveDecryptResult, throwPreservingCode } from '@/dynamodb/helpers' +import { logger } from '@/utils/logger' describe('resolveDecryptResult', () => { it('awaits a plain promise when the operation has no .audit (typed client)', async () => { @@ -42,6 +43,47 @@ describe('resolveDecryptResult', () => { resolveDecryptResult(Promise.resolve(failure), { metadata: {} }), ).resolves.toEqual(failure) }) + + it('returns a failure — not a silent undefined success — for a malformed result', async () => { + // A non-conforming client that resolves to a bare value (or `{}`) has + // neither `data` nor `failure`. Casting it straight through would surface a + // fake success carrying `undefined`; the shape must be rejected instead. + for (const malformed of [{}, 42, undefined]) { + const result = await resolveDecryptResult(Promise.resolve(malformed), {}) + + expect(result.failure).toBeDefined() + expect(typeof result.failure?.message).toBe('string') + expect(result.data).toBeUndefined() + } + }) + + it('logs when audit metadata is dropped for a non-chainable operation', async () => { + const spy = vi.spyOn(logger, 'debug').mockImplementation(() => {}) + + try { + await resolveDecryptResult(Promise.resolve({ data: { x: 1 } }), { + metadata: { m: 42 }, + }) + + expect(spy).toHaveBeenCalledWith( + expect.stringContaining('audit metadata ignored'), + ) + } finally { + spy.mockRestore() + } + }) + + it('does not log about audit metadata when none is passed', async () => { + const spy = vi.spyOn(logger, 'debug').mockImplementation(() => {}) + + try { + await resolveDecryptResult(Promise.resolve({ data: { x: 1 } }), {}) + + expect(spy).not.toHaveBeenCalled() + } finally { + spy.mockRestore() + } + }) }) describe('throwPreservingCode', () => { diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index a7d89e8a..06c6049b 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -3,6 +3,7 @@ import { ProtectError as FfiProtectError } from '@cipherstash/protect-ffi' import { resolveEncryptColumnMap } from '@/encryption/helpers/model-helpers' import type { AnyV3Table } from '@/eql/v3' import type { EncryptedValue } from '@/types' +import { hasBuildColumnKeyMap } from '@/types' import { logger } from '@/utils/logger' import type { AnyEncryptedTable, EncryptedDynamoDBError } from './types' @@ -18,10 +19,7 @@ export const searchTermAttrSuffix = '__hmac' * as *the* signal. Only v3 tables define it. */ export function isV3Table(table: AnyEncryptedTable): table is AnyV3Table { - return ( - 'buildColumnKeyMap' in table && - typeof table.buildColumnKeyMap === 'function' - ) + return hasBuildColumnKeyMap(table) } export class EncryptedDynamoDBErrorImpl @@ -119,6 +117,22 @@ export async function resolveDecryptResult( ? await chainable.audit(auditData) : await operation + // A conforming client resolves to `{ data }` or `{ failure }`. A bare value + // (or wrong shape) has neither, so casting it straight through would surface a + // fake success carrying `undefined`. Reject it as a failure instead. + if ( + resolved === null || + typeof resolved !== 'object' || + (!('data' in resolved) && !('failure' in resolved)) + ) { + return { + failure: { + message: + 'DynamoDB: decrypt returned a malformed result — expected { data } or { failure }.', + }, + } + } + return resolved as | { data: T; failure?: never } | { data?: never; failure: DecryptFailure } @@ -149,8 +163,10 @@ export function throwPreservingCode(failure: { * `Date` values — making the whole `types.Timestamp*` / `types.Date*` domain * family unusable through this adapter — and blew the stack on a circular * reference. `structuredClone` handles Date, Map, Set, TypedArray and cycles - * natively. Values it cannot structurally clone fall back to the original - * reference rather than throwing. + * natively. Values it cannot structurally clone (a function/symbol/WeakMap-valued + * property) fall back to a per-key shallow copy into a FRESH object rather than + * throwing — the offending value passes through by reference, every other key + * is copied, so the caller's original is never handed to the FFI. */ export function deepClone(obj: T): T { if (obj === null || typeof obj !== 'object') { @@ -160,7 +176,16 @@ export function deepClone(obj: T): T { try { return structuredClone(obj) } catch { - return obj + // `structuredClone` rejects function/symbol/WeakMap-valued properties. Fall + // back to a per-key shallow copy into a FRESH object so the caller's + // original is never handed to the FFI — otherwise the docblock guarantee + // "encryption never mutates a caller's object" is silently voided. The + // offending value passes through by reference; every other key is copied, + // and a class instance is flattened to a plain object (prototype dropped). + if (Array.isArray(obj)) { + return [...obj] as T + } + return { ...(obj as Record) } as T } } @@ -263,7 +288,8 @@ export function toEncryptedDynamoItem( ).reduce( (acc, [key, val]) => { const processed = processValue(key, val, true) - return Object.assign({}, acc, processed) + Object.assign(acc, processed) + return acc }, {} as Record, ) @@ -277,7 +303,8 @@ export function toEncryptedDynamoItem( return Object.entries(encrypted).reduce( (putItem, [attrName, attrValue]) => { const processed = processValue(attrName, attrValue, false) - return Object.assign({}, putItem, processed) + Object.assign(putItem, processed) + return putItem }, {} as Record, ) @@ -346,7 +373,6 @@ export function toItemWithEqlPayloads( function processValue( attrName: string, attrValue: unknown, - isNested: boolean, prefix = '', ): Record { if (attrValue === null || attrValue === undefined) { @@ -370,6 +396,17 @@ export function toItemWithEqlPayloads( // under a `details` group), so both authoring conventions resolve. const matched = matchColumn(columnName, prefix) + // A stored ciphertext attribute that names no declared column is almost + // always a schema rename: the value reads back as raw base64 with no error. + // Leave the passthrough behaviour intact (an unmatched `*__source` is not an + // envelope), but make the silent drop observable. + if (attrName.endsWith(ciphertextAttrSuffix) && !matched) { + const dotted = prefix ? `${prefix}.${attrName}` : attrName + logger.debug( + `DynamoDB: attribute "${dotted}" ends in ${ciphertextAttrSuffix} but names no declared column — passing it through as raw ciphertext (a column rename can cause this).`, + ) + } + // Handle encrypted payload. An unmatched attribute is NOT an envelope, even // when nested — previously `|| isNested` made every nested `*__source` a // decrypt target, so an unrelated customer attribute was handed to the FFI. @@ -422,10 +459,10 @@ export function toItemWithEqlPayloads( const processed = processValue( key, val, - true, prefix ? `${prefix}.${attrName}` : attrName, ) - return Object.assign({}, acc, processed) + Object.assign(acc, processed) + return acc }, {} as Record, ) @@ -438,8 +475,9 @@ export function toItemWithEqlPayloads( return Object.entries(decrypted).reduce( (formattedItem, [attrName, attrValue]) => { - const processed = processValue(attrName, attrValue, false) - return Object.assign({}, formattedItem, processed) + const processed = processValue(attrName, attrValue) + Object.assign(formattedItem, processed) + return formattedItem }, {} as Record, ) diff --git a/packages/stack/src/dynamodb/index.ts b/packages/stack/src/dynamodb/index.ts index e415be06..bce47df8 100644 --- a/packages/stack/src/dynamodb/index.ts +++ b/packages/stack/src/dynamodb/index.ts @@ -1,4 +1,5 @@ import type { EncryptedValue } from '@/types' +import { isV3Table } from './helpers' import { BulkDecryptModelsOperation } from './operations/bulk-decrypt-models' import { BulkEncryptModelsOperation } from './operations/bulk-encrypt-models' import { DecryptModelOperation } from './operations/decrypt-model' @@ -9,6 +10,57 @@ import type { EncryptedDynamoDBInstance, } from './types' +/** + * Fail fast on an EQL version mismatch between the client and the table. + * + * A v3 table encrypted through a client that is NOT in EQL v3 mode for it — a + * v2-mode client, or one initialised for a different schema set — otherwise + * surfaces only much later as an opaque FFI deserialization error, far from the + * misconfiguration that caused it. + * + * The client does not expose its EQL wire version directly (it is baked into the + * FFI client at `newClient` time and neither the nominal `EncryptionClient` nor + * the typed `EncryptionV3` wrapper re-surfaces it). The reliable, public signal + * both client shapes DO expose is `getEncryptConfig()`: a v3 table can only be + * encrypted by a client that registered it, and a client built for v3 registers + * it under its `tableName`. So the guard fires only when we can PROVE the table + * is absent from a readable config — never on an unreadable one, to avoid a + * false positive. + * + * This runs at the first point where both the client and a concrete table are in + * hand: the operation methods below, when a table is supplied. `encryptedDynamoDB` + * itself receives no table, so it cannot check earlier. + * + * RESIDUAL GAP: a client explicitly forced to `eqlVersion: 2` over a v3 schema + * set (a deliberate migration path) DOES register the table yet emits v2 wire; + * that is not detectable without a wire-version accessor the client does not + * provide, so it is out of scope here. + */ +function assertClientTableVersionMatch( + encryptionClient: EncryptedDynamoDBConfig['encryptionClient'], + table: AnyEncryptedTable, +): void { + // Only v3 tables carry the strict wire-format requirement this guards. + if (!isV3Table(table)) return + + const getEncryptConfig = ( + encryptionClient as { + getEncryptConfig?: () => { tables?: Record } | undefined + } + ).getEncryptConfig + + // Without a readable config we cannot prove a mismatch — stay silent rather + // than false-positive on a client shape that does not expose it. + if (typeof getEncryptConfig !== 'function') return + const config = getEncryptConfig.call(encryptionClient) + if (!config?.tables) return + if (table.tableName in config.tables) return + + throw new Error( + `encryptedDynamoDB: EQL version mismatch — the EQL v3 table "${table.tableName}" is not registered with this encryption client, so the client is not in EQL v3 mode for it. A v3 table requires a v3-mode client: build it with EncryptionV3({ schemas: [
] }) (or Encryption({ schemas: [
], config: { eqlVersion: 3 } })) and pass that client to encryptedDynamoDB. Otherwise encrypt/decrypt fails later inside the FFI with an opaque deserialization error.`, + ) +} + /** * Create an encrypted DynamoDB helper bound to an `EncryptionClient`. * @@ -66,61 +118,73 @@ export function encryptedDynamoDB( // The public interface is OVERLOADED per wire version (a v3 overload that // knows the storage split, a v2 one that does not). A single generic - // implementation signature cannot be inferred as satisfying both arms, so the - // instance is built against the erased shape and asserted once, here. Nothing - // about the runtime differs between the overloads — the table decides - // everything, and it does so at runtime. - const instance = { - encryptModel>( - item: T, - table: AnyEncryptedTable, - ) { - return new EncryptModelOperation( - encryptionClient, - item, - table, - options, - ) - }, + // implementation signature cannot be inferred as satisfying both arms — the + // v3 overload's return is the derived `EncryptedAttributes` storage split, + // which the erased implementation (built against `AnyEncryptedTable`) cannot + // reproduce. So each method is written once against the erased shape and cast + // to ITS OWN interface member, and the assembled object is annotated with the + // public type. Nothing about the runtime differs between the overloads — the + // table decides everything, and it does so at runtime. + // + // Why per-method casts rather than one object-level `as`: the object literal + // is checked against the annotation, so a method renamed/added/removed on the + // interface (drift between the four impls and the eight-overload public type) + // now fails to compile — a single `as EncryptedDynamoDBInstance` erased that. + // What a per-method cast still cannot check is the v3 RETURN precision + // (`EncryptedAttributes`), for the erasure reason above; that surface is + // locked instead by `__tests__/dynamodb/client-compat.test-d.ts`. + const encryptModel = >( + item: T, + table: AnyEncryptedTable, + ) => { + assertClientTableVersionMatch(encryptionClient, table) + return new EncryptModelOperation(encryptionClient, item, table, options) + } - bulkEncryptModels>( - items: T[], - table: AnyEncryptedTable, - ) { - return new BulkEncryptModelsOperation( - encryptionClient, - items, - table, - options, - ) - }, + const bulkEncryptModels = >( + items: T[], + table: AnyEncryptedTable, + ) => { + assertClientTableVersionMatch(encryptionClient, table) + return new BulkEncryptModelsOperation( + encryptionClient, + items, + table, + options, + ) + } + + const decryptModel = >( + item: Record, + table: AnyEncryptedTable, + ) => { + assertClientTableVersionMatch(encryptionClient, table) + return new DecryptModelOperation(encryptionClient, item, table, options) + } - decryptModel>( - item: Record, - table: AnyEncryptedTable, - ) { - return new DecryptModelOperation( - encryptionClient, - item, - table, - options, - ) - }, + const bulkDecryptModels = >( + items: Record[], + table: AnyEncryptedTable, + ) => { + assertClientTableVersionMatch(encryptionClient, table) + return new BulkDecryptModelsOperation( + encryptionClient, + items, + table, + options, + ) + } - bulkDecryptModels>( - items: Record[], - table: AnyEncryptedTable, - ) { - return new BulkDecryptModelsOperation( - encryptionClient, - items, - table, - options, - ) - }, + const instance: EncryptedDynamoDBInstance = { + encryptModel: encryptModel as EncryptedDynamoDBInstance['encryptModel'], + bulkEncryptModels: + bulkEncryptModels as EncryptedDynamoDBInstance['bulkEncryptModels'], + decryptModel: decryptModel as EncryptedDynamoDBInstance['decryptModel'], + bulkDecryptModels: + bulkDecryptModels as EncryptedDynamoDBInstance['bulkDecryptModels'], } - return instance as EncryptedDynamoDBInstance + return instance } export type { AuditConfig } from './operations/base-operation' diff --git a/packages/stack/src/encryption/index.ts b/packages/stack/src/encryption/index.ts index ae36a7e0..a66817e7 100644 --- a/packages/stack/src/encryption/index.ts +++ b/packages/stack/src/encryption/index.ts @@ -29,6 +29,7 @@ import type { Plaintext, ScalarQueryTerm, } from '@/types' +import { hasBuildColumnKeyMap } from '@/types' import { logger } from '@/utils/logger' import { toFfiKeysetIdentifier } from './helpers' import { isScalarQueryTermArray } from './helpers/type-guards' @@ -88,9 +89,7 @@ export function resolveEqlVersion( schemas: readonly BuildableTable[], explicit?: 2 | 3, ): 2 | 3 | undefined { - const v3Count = schemas.filter( - (schema) => typeof schema.buildColumnKeyMap === 'function', - ).length + const v3Count = schemas.filter(hasBuildColumnKeyMap).length if (v3Count > 0 && v3Count < schemas.length) { throw new Error( diff --git a/packages/stack/src/types.ts b/packages/stack/src/types.ts index fdb0ff4d..c516d848 100644 --- a/packages/stack/src/types.ts +++ b/packages/stack/src/types.ts @@ -243,6 +243,24 @@ export interface BuildableTable { buildColumnKeyMap?(): Record } +/** + * The canonical EQL v3 marker: v3 tables expose `buildColumnKeyMap()`, v2 tables + * don't. This is the single predicate the codebase uses to decide which wire + * version a table targets — a second hand-written spelling of the check is how a + * v2 envelope eventually gets built for a v3 table once the marker drifts, so + * every site (`encryption/index.ts`, `wasm-inline.ts`, `dynamodb/helpers.ts`) + * routes through here. + */ +export function hasBuildColumnKeyMap( + table: T, +): table is T & { buildColumnKeyMap: () => Record } { + return ( + 'buildColumnKeyMap' in table && + typeof (table as { buildColumnKeyMap?: unknown }).buildColumnKeyMap === + 'function' + ) +} + export type EncryptionClientConfig = { schemas: AtLeastOneCsTable config?: ClientConfig diff --git a/packages/stack/src/wasm-inline.ts b/packages/stack/src/wasm-inline.ts index a617a4b8..a3409a8f 100644 --- a/packages/stack/src/wasm-inline.ts +++ b/packages/stack/src/wasm-inline.ts @@ -113,6 +113,7 @@ import type { Plaintext, QueryTypeName, } from '@/types' +import { hasBuildColumnKeyMap } from '@/types' // ----------------------------------------------------------------------- // Schema + type re-exports @@ -982,9 +983,7 @@ export async function Encryption( // `buildColumnKeyMap` marker) with a clear message rather than pinning the // client to v3 wire against a v2 schema and failing opaquely inside the FFI. for (const table of schemas) { - const isV3 = - typeof (table as { buildColumnKeyMap?: unknown }).buildColumnKeyMap === - 'function' + const isV3 = hasBuildColumnKeyMap(table) if (!isV3) { throw new Error( '[encryption]: `@cipherstash/stack/wasm-inline` is EQL v3 only — author schemas with `types` / `encryptedTable` from this entry. (EQL v2 is available on the native `@cipherstash/stack` entry.)', diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index 9c2b8614..fa952a1d 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -55,6 +55,8 @@ Non-encrypted attributes pass through unchanged. On decryption, the `__source` a There is no infrastructure migration between them — DynamoDB has no EQL extension to install and no schema to alter — but there is no automatic *data* migration path either. The two write **different wire formats**, so items written under one version cannot be decrypted by the other. Pick one per table and stay on it; to switch an existing table you must re-encrypt every item. +The client must be built for the table. Build the client with the same v3 table you hand to `encryptedDynamoDB` — `EncryptionV3({ schemas: [users] })` (or `Encryption({ schemas: [users], config: { eqlVersion: 3 } })`). Passing a v3 table to a client that never registered it (a client built for a different schema set) throws a clear error naming the table on the first operation, instead of failing later with an opaque FFI deserialization error. + **Nested attributes work in both versions**, with different authoring syntax. DynamoDB items are natively nested, so this matters here more than on Postgres. Both versions encrypt *selected leaves in place*: the item keeps its shape and unlisted siblings stay plaintext. @@ -76,18 +78,24 @@ const users = encryptedTable("users", { }) ``` -Both produce the same DynamoDB attribute *layout* — and the v3 form keeps equality queryability on the nested attribute — but the encrypted contents are version-specific: an item written under one version cannot be decrypted under the other (see the compatibility note above). +Both produce the same nested DynamoDB attribute *layout*, but the encrypted contents are version-specific: an item written under one version cannot be decrypted under the other (see the compatibility note above). ```jsonc { "pk": "u#1", "profile": { "ssn__source": "", - "ssn__hmac": "", // key-condition on profile.ssn__hmac + "ssn__hmac": "", // equality term — FilterExpression only, not a key condition "note__source": "", "city": "Sydney" // not in schema, stays plaintext } } ``` +> A nested equality term like `profile.ssn__hmac` lives *inside* the `profile` +> map, so it can only be matched with a `FilterExpression` — DynamoDB key +> conditions and secondary-index keys must be top-level scalar attributes. If you +> need the HMAC to back a key condition or GSI, declare the field as a top-level +> column (`ssn: types.TextEq("ssn")`) so it is stored as a top-level `ssn__hmac`. + > The dotted string is the *property key* as well as the column name — the model > is matched by dotted path, so `{ profile: { ssn } }` resolves correctly. From 75b1558c9f1f66c9e7133366cd68b3e4cf478150 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 21 Jul 2026 21:46:39 +1000 Subject: [PATCH 15/20] fix(stack): adapt DynamoDB v3 adapter to protect-ffi 0.30 SteVec format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After rebasing onto main (protect-ffi 0.29 → 0.30), two 0.30 changes surfaced: - SteVec (searchable JSON) documents now carry a required per-document KeyHeader `h` (`{v, k, i, h, sv}`), and 0.30 decrypt is strict — a document missing `h` fails with "missing field `h`". The adapter split a JSON payload down to its `sv` entries only, discarding `h`, so v3 JSON round-trips broke. It now stores `{ h, sv }` under `__source` (the non-reconstructable parts) and rebuilds `{v, i, k: 'sv', h, sv}` on read — `v`/`i`/`k` stay reconstructed so no envelope metadata leaks into storage (invariant 6 holds). - protect-ffi 0.30 dropped EQL v2 `searchableJson` (SteVec) entirely; `resolveEqlVersion` now throws for a v2 ste_vec schema. Removed the v2 `searchableJson` column and its cases from the v2 characterisation suites (encrypted-dynamodb.test.ts, helpers.test.ts) — the capability no longer exists. v3 JSON (`types.Json`) is the supported path and is covered by the helpers-v3 / properties SteVec tests, updated to pin the `h` KeyHeader. Pure mapping tests (helpers, helpers-v3, properties) and test:types pass locally; the live v3/v2 suites run under 0.30 in CI (the 0.30 CTS auth endpoint is unreachable from the local sandbox, so live paths are CI-verified). --- .../dynamodb/client-compat.test-d.ts | 9 +- .../dynamodb/encrypted-dynamodb-v3.test.ts | 11 +- .../dynamodb/encrypted-dynamodb.test.ts | 29 --- .../__tests__/dynamodb/helpers-v3.test.ts | 35 +++- .../stack/__tests__/dynamodb/helpers.test.ts | 30 --- .../__tests__/dynamodb/properties.test.ts | 184 ++++++++++-------- packages/stack/src/dynamodb/helpers.ts | 39 ++-- packages/stack/src/dynamodb/types.ts | 10 +- 8 files changed, 184 insertions(+), 163 deletions(-) diff --git a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts index ea032ae9..dc46d0c2 100644 --- a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts +++ b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts @@ -154,14 +154,19 @@ describe('the v3 overload types the DynamoDB storage split', () => { expectTypeOf(result.data.role).toEqualTypeOf() }) - it('types a JSON document as its stored ste_vec array', async () => { + it('types a JSON document as its stored ste_vec entries plus KeyHeader', async () => { const result = await dynamo.encryptModel( { pk: 'a', meta: { a: 1 } }, usersV3, ) if (result.failure) return - expectTypeOf(result.data.meta__source).toEqualTypeOf() + // A SteVec document is stored as `{ h, sv }` — the entries plus the + // per-document KeyHeader `h` that protect-ffi 0.30 decrypt requires. + expectTypeOf(result.data.meta__source).toEqualTypeOf<{ + h: unknown + sv: unknown[] + }>() expectTypeOf(result.data).not.toHaveProperty('meta__hmac') }) diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index 04c73cc3..d94d2314 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -147,7 +147,7 @@ describe.skipIf(!hasCipherStashCreds)('encryptModel with a v3 table', () => { ]) }) - it('stores a Json domain as a ste_vec array', async () => { + it('stores a Json domain as its sv entries plus the KeyHeader', async () => { const result = await typedDynamo.encryptModel( { pk: 'user#5', meta: { a: 1, b: { c: 'deep' } } }, users, @@ -155,8 +155,13 @@ describe.skipIf(!hasCipherStashCreds)('encryptModel with a v3 table', () => { if (result.failure) throw new Error(result.failure.message) - expect(Array.isArray(result.data.meta__source)).toBe(true) - expect(result.data.meta__source.length).toBeGreaterThan(0) + // A SteVec document is stored as `{ h, sv }` under __source: the `sv` + // entries plus the per-document KeyHeader `h` that protect-ffi 0.30 decrypt + // requires. `v`/`i`/`k` are reconstructed on read, so they are not stored. + const stored = result.data.meta__source as { h: unknown; sv: unknown[] } + expect(stored.h).toBeDefined() + expect(Array.isArray(stored.sv)).toBe(true) + expect(stored.sv.length).toBeGreaterThan(0) }) it('passes a null column value through without splitting it', async () => { diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts index 3b7d8f51..cecad04b 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb.test.ts @@ -24,7 +24,6 @@ import { encryptedColumn, encryptedField, encryptedTable } from '@/schema' const users = encryptedTable('users', { email: encryptedColumn('email').equality(), name: encryptedColumn('name'), - doc: encryptedColumn('doc').dataType('json').searchableJson(), example: { protected: encryptedField('example.protected'), }, @@ -34,7 +33,6 @@ type User = { pk: string email?: string | null name?: string | null - doc?: Record role?: string example?: { protected?: string | null; notProtected?: string } } @@ -55,7 +53,6 @@ type StoredUser = { email__source?: string email__hmac?: string name__source?: string - doc__source?: unknown[] example?: { protected__source?: string; notProtected?: string } } @@ -116,20 +113,6 @@ describe('encryptModel', () => { expect(result.data).not.toHaveProperty('name__hmac') }) - it('stores a searchableJson column as a ste_vec array in __source', async () => { - const result = await dynamo.encryptModel( - { pk: 'user#3', doc: { a: 1, b: 'two' } }, - users, - ) - - if (result.failure) throw new Error(result.failure.message) - - expect(Array.isArray(storedAttrs(result.data).doc__source)).toBe(true) - expect( - (storedAttrs(result.data).doc__source as unknown[]).length, - ).toBeGreaterThan(0) - }) - it('encrypts a nested field in place, keeping siblings plaintext', async () => { const result = await dynamo.encryptModel( { @@ -188,18 +171,6 @@ describe('decryptModel', () => { expect(decrypted.data).toEqual(original) }) - it('round-trips a searchableJson document', async () => { - const original: User = { pk: 'user#8', doc: { a: 1, b: 'two' } } - - const encrypted = await dynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) - - const decrypted = await dynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) - - expect(decrypted.data).toEqual(original) - }) - it('tolerates the __hmac attribute being present on the stored item', async () => { const encrypted = await dynamo.encryptModel( { pk: 'user#9', email: 'frank@example.com' }, diff --git a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts index 3ce87b9a..96a7552f 100644 --- a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts @@ -127,14 +127,25 @@ describe('toEncryptedDynamoItem with v3 payloads', () => { }) }) - it('stores a v3 ste_vec document as its sv array', () => { + it('stores a v3 ste_vec document as its entries plus the KeyHeader', () => { const entries = [{ s: 'sel', c: 'ct', a: false, hm: 'h' }] + // Only the non-reconstructable parts are stored: the `sv` entries and the + // per-document KeyHeader `h` (protect-ffi 0.30 decrypt requires `h`). The + // `v`/`i`/`k` envelope fields are rebuilt on read, so they are not stored. const result = toEncryptedDynamoItem( - { meta: { v: 3, k: 'sv', i: { t: 'users', c: 'meta' }, sv: entries } }, + { + meta: { + v: 3, + k: 'sv', + i: { t: 'users', c: 'meta' }, + h: 'key-header', + sv: entries, + }, + }, encryptedAttrs, ) - expect(result).toEqual({ meta__source: entries }) + expect(result).toEqual({ meta__source: { h: 'key-header', sv: entries } }) }) }) @@ -148,12 +159,24 @@ describe('toItemWithEqlPayloads for a v3 table', () => { expect(result.email).not.toHaveProperty('k') }) - it('rebuilds a v3 ste_vec envelope, keeping the mandatory k tag', () => { + it('rebuilds a v3 ste_vec envelope, restoring k and the KeyHeader', () => { + // The stored __source is `{ h, sv }`; the read path rebuilds the full + // envelope, restoring the `k` tag and the `h` KeyHeader that 0.30 decrypt + // requires, with `i`/`v` reconstructed from the schema. const entries = [{ s: 'sel', c: 'ct' }] - const result = toItemWithEqlPayloads({ meta__source: entries }, users) + const result = toItemWithEqlPayloads( + { meta__source: { h: 'key-header', sv: entries } }, + users, + ) expect(result).toEqual({ - meta: { i: { c: 'meta', t: 'users' }, v: 3, k: 'sv', sv: entries }, + meta: { + i: { c: 'meta', t: 'users' }, + v: 3, + k: 'sv', + h: 'key-header', + sv: entries, + }, }) }) diff --git a/packages/stack/__tests__/dynamodb/helpers.test.ts b/packages/stack/__tests__/dynamodb/helpers.test.ts index 85ce56cf..5a133093 100644 --- a/packages/stack/__tests__/dynamodb/helpers.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers.test.ts @@ -25,7 +25,6 @@ const users = encryptedTable('users', { email: encryptedColumn('email').equality(), name: encryptedColumn('name'), blob: encryptedColumn('blob').dataType('json'), - doc: encryptedColumn('doc').dataType('json').searchableJson(), example: { protected: encryptedField('example.protected'), }, @@ -42,14 +41,6 @@ const ct = (c: string, hm?: string) => ({ ...(hm ? { hm } : {}), }) -/** A minimal v2 ste_vec payload as the FFI returns it. */ -const sv = (entries: unknown[]) => ({ - k: 'sv' as const, - v: 2, - i: { t: 'users', c: 'doc' }, - sv: entries, -}) - describe('attribute suffixes', () => { it('are the documented DynamoDB naming convention', () => { expect(ciphertextAttrSuffix).toBe('__source') @@ -83,13 +74,6 @@ describe('toEncryptedDynamoItem (write path)', () => { expect(result).not.toHaveProperty(`name${searchTermAttrSuffix}`) }) - it('stores the ste_vec array in __source for a searchable JSON payload', () => { - const entries = [{ s: 'sel', t: 'term' }] - const result = toEncryptedDynamoItem({ doc: sv(entries) }, encryptedAttrs) - - expect(result).toEqual({ doc__source: entries }) - }) - it('passes attributes absent from the schema through untouched', () => { const item = { pk: 'user#1', @@ -168,20 +152,6 @@ describe('toItemWithEqlPayloads (read path)', () => { expect(result.email).not.toHaveProperty('hm') }) - it('rebuilds a ste_vec envelope for a searchableJson column', () => { - const entries = [{ s: 'sel', t: 'term' }] - const result = toItemWithEqlPayloads({ doc__source: entries }, users) - - expect(result).toEqual({ - doc: { - i: { c: 'doc', t: 'users' }, - v: 2, - k: 'sv', - sv: entries, - }, - }) - }) - it('rebuilds a scalar envelope for a JSON column without a ste_vec index', () => { const result = toItemWithEqlPayloads({ blob__source: 'ciphertext' }, users) diff --git a/packages/stack/__tests__/dynamodb/properties.test.ts b/packages/stack/__tests__/dynamodb/properties.test.ts index 89b8cf7f..a2873bac 100644 --- a/packages/stack/__tests__/dynamodb/properties.test.ts +++ b/packages/stack/__tests__/dynamodb/properties.test.ts @@ -139,6 +139,9 @@ const roundTripCase = fc ), cts: fc.tuple(...names.map(() => ciphertext)), svs: fc.tuple(...names.map(() => steVecEntries)), + // The per-document SteVec KeyHeader (`h`). Opaque to the mapping, but + // protect-ffi 0.30 decrypt requires it, so it must survive the round trip. + hs: fc.tuple(...names.map(() => fc.string({ minLength: 1 }))), hms: fc.tuple( ...names.map(() => fc.option(fc.string({ minLength: 1 }), { nil: undefined }), @@ -151,62 +154,71 @@ const roundTripCase = fc describe('property: write → read round trip', () => { it('preserves every ciphertext and rebuilds the envelope, for any column set', () => { fc.assert( - fc.property(roundTripCase, ({ names, kinds, cts, svs, hms, plain }) => { - const table = encryptedTable( - 'users', - Object.fromEntries( - names.map((n, idx) => [ - n, - kinds[idx] === 'json' ? types.Json(n) : types.TextEq(n), - ]), - ), - ) - const encryptedAttrs = Object.keys(table.buildColumnKeyMap()) - - const item: Record = { [PLAINTEXT_KEY]: plain } - const expectedStored: Record = { - [PLAINTEXT_KEY]: plain, - } - const expectedRebuilt: Record = { - [PLAINTEXT_KEY]: plain, - } + fc.property( + roundTripCase, + ({ names, kinds, cts, svs, hs, hms, plain }) => { + const table = encryptedTable( + 'users', + Object.fromEntries( + names.map((n, idx) => [ + n, + kinds[idx] === 'json' ? types.Json(n) : types.TextEq(n), + ]), + ), + ) + const encryptedAttrs = Object.keys(table.buildColumnKeyMap()) + + const item: Record = { [PLAINTEXT_KEY]: plain } + const expectedStored: Record = { + [PLAINTEXT_KEY]: plain, + } + const expectedRebuilt: Record = { + [PLAINTEXT_KEY]: plain, + } - names.forEach((name, idx) => { - const i = { t: 'users', c: name } - if (kinds[idx] === 'json') { - item[name] = { v: 3, k: 'sv', i, sv: svs[idx] } - expectedStored[`${name}${ciphertextAttrSuffix}`] = svs[idx] + names.forEach((name, idx) => { + const i = { t: 'users', c: name } + if (kinds[idx] === 'json') { + // A SteVec document stores its `sv` entries plus the KeyHeader `h` + // (0.30 decrypt requires `h`); `v`/`i`/`k` are rebuilt on read. + item[name] = { v: 3, k: 'sv', i, h: hs[idx], sv: svs[idx] } + expectedStored[`${name}${ciphertextAttrSuffix}`] = { + h: hs[idx], + sv: svs[idx], + } + expectedRebuilt[name] = { + i: { c: name, t: 'users' }, + v: 3, + k: 'sv', + h: hs[idx], + sv: svs[idx], + } + return + } + item[name] = { + v: 3, + i, + c: cts[idx], + ...(hms[idx] === undefined ? {} : { hm: hms[idx] }), + } + expectedStored[`${name}${ciphertextAttrSuffix}`] = cts[idx] + if (hms[idx] !== undefined) { + expectedStored[`${name}${searchTermAttrSuffix}`] = hms[idx] + } + // The search term is a write-side index, not part of the envelope: + // the read path drops it rather than round-tripping it. expectedRebuilt[name] = { i: { c: name, t: 'users' }, v: 3, - k: 'sv', - sv: svs[idx], + c: cts[idx], } - return - } - item[name] = { - v: 3, - i, - c: cts[idx], - ...(hms[idx] === undefined ? {} : { hm: hms[idx] }), - } - expectedStored[`${name}${ciphertextAttrSuffix}`] = cts[idx] - if (hms[idx] !== undefined) { - expectedStored[`${name}${searchTermAttrSuffix}`] = hms[idx] - } - // The search term is a write-side index, not part of the envelope: - // the read path drops it rather than round-tripping it. - expectedRebuilt[name] = { - i: { c: name, t: 'users' }, - v: 3, - c: cts[idx], - } - }) + }) - const stored = toEncryptedDynamoItem(item, encryptedAttrs) - expect(stored).toEqual(expectedStored) - expect(toItemWithEqlPayloads(stored, table)).toEqual(expectedRebuilt) - }), + const stored = toEncryptedDynamoItem(item, encryptedAttrs) + expect(stored).toEqual(expectedStored) + expect(toItemWithEqlPayloads(stored, table)).toEqual(expectedRebuilt) + }, + ), ) }) }) @@ -348,20 +360,27 @@ describe('property: the rebuilt wire version follows the table', () => { ) }) - it('a v3 JSON column keeps v: 3 with the mandatory k: "sv"', () => { + it('a v3 JSON column keeps v: 3, the mandatory k: "sv", and the KeyHeader', () => { fc.assert( - fc.property(safeName, steVecEntries, (name, entries) => { - const table = encryptedTable('t', { [name]: types.Json(name) }) - - const rebuilt = toItemWithEqlPayloads( - { [`${name}${ciphertextAttrSuffix}`]: entries }, - table, - )[name] as Record - - expect(rebuilt.v).toBe(3) - expect(rebuilt.k).toBe('sv') - expect(rebuilt.sv).toEqual(entries) - }), + fc.property( + safeName, + steVecEntries, + fc.string({ minLength: 1 }), + (name, entries, h) => { + const table = encryptedTable('t', { [name]: types.Json(name) }) + + const rebuilt = toItemWithEqlPayloads( + { [`${name}${ciphertextAttrSuffix}`]: { h, sv: entries } }, + table, + )[name] as Record + + expect(rebuilt.v).toBe(3) + expect(rebuilt.k).toBe('sv') + // The per-document KeyHeader survives — 0.30 decrypt requires it. + expect(rebuilt.h).toBe(h) + expect(rebuilt.sv).toEqual(entries) + }, + ), ) }) @@ -459,21 +478,32 @@ describe('property: no envelope metadata reaches storage', () => { ) }) - it('a ste_vec document is stored as its bare sv array', () => { + it('a ste_vec document is stored as its entries plus KeyHeader, no envelope metadata', () => { fc.assert( - fc.property(safeName, steVecEntries, (name, entries) => { - const table = encryptedTable('t', { [name]: types.Json(name) }) - const stored = toEncryptedDynamoItem( - { - [name]: { v: 3, k: 'sv', i: { t: 't', c: name }, sv: entries }, - }, - Object.keys(table.buildColumnKeyMap()), - ) - - expect(Object.keys(stored)).toEqual([`${name}${ciphertextAttrSuffix}`]) - expect(stored[`${name}${ciphertextAttrSuffix}`]).toEqual(entries) - expect(hasEnvelopeMetadata(stored)).toBe(false) - }), + fc.property( + safeName, + steVecEntries, + fc.string({ minLength: 1 }), + (name, entries, h) => { + const table = encryptedTable('t', { [name]: types.Json(name) }) + const stored = toEncryptedDynamoItem( + { + [name]: { v: 3, k: 'sv', i: { t: 't', c: name }, h, sv: entries }, + }, + Object.keys(table.buildColumnKeyMap()), + ) + + expect(Object.keys(stored)).toEqual([ + `${name}${ciphertextAttrSuffix}`, + ]) + expect(stored[`${name}${ciphertextAttrSuffix}`]).toEqual({ + h, + sv: entries, + }) + // `v`/`i`/`k` are reconstructed on read, never stored. + expect(hasEnvelopeMetadata(stored)).toBe(false) + }, + ), ) }) }) diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index 06c6049b..66ee114c 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -212,6 +212,11 @@ type StoredEqlPayload = { hm?: unknown /** ste_vec entries for a JSON document. */ sv?: unknown + /** + * Per-document SteVec KeyHeader. Present on a v3 JSON (`k: 'sv'`) document; + * protect-ffi 0.30 decrypt requires it, so it is stored alongside `sv`. + */ + h?: unknown } /** @@ -253,12 +258,18 @@ export function toEncryptedDynamoItem( ) { const encryptPayload = attrValue - // A JSON document, in either wire version, keeps its `k: 'sv'` tag. Its - // index terms live *inside* the `sv` entries, so the whole array is - // stored and there is no separate search-term attribute to split out. + // A JSON document keeps its `k: 'sv'` tag. Its index terms live *inside* + // the `sv` entries, so there is no separate search-term attribute to split + // out. Store the `sv` entries together with the per-document KeyHeader + // `h`: protect-ffi 0.30's SteVec decrypt requires `h` (there is no root + // `c` to reconstruct from) and it is not derivable, whereas `v`/`i`/`k` + // ARE reconstructed on read and so are kept out of the stored attribute. if (encryptPayload?.k === 'sv' && encryptPayload.sv) { const result: Record = {} - result[`${attrName}${ciphertextAttrSuffix}`] = encryptPayload.sv + result[`${attrName}${ciphertextAttrSuffix}`] = { + h: encryptPayload.h, + sv: encryptPayload.sv, + } return result } @@ -416,22 +427,24 @@ export function toItemWithEqlPayloads( // `emailAddress: types.TextEq('email_address')`. const i = { c: toColumnName(matched), t: encryptConfig.tableName } - // A JSON document is stored as its ste_vec array and must be rebuilt with - // `k: 'sv'`. Look the config up by the resolved identifier so a nested - // JSON column (registered under a dotted path) is detected too — keyed on - // the leaf it would be missing, and would be rebuilt as a scalar. - // A v3 column builds the same `{ cast_as, indexes }` shape as a v2 one, - // so this detection needs no version branch. + // A JSON document is stored as `{ h, sv }` (see the write path); rebuild + // the full SteVec envelope around it. protect-ffi 0.30 deserialization is + // strict — it requires `k` ("missing field `k`") and the per-document + // KeyHeader `h` ("missing field `h`"), and there is no root `c`. `i`/`v`/`k` + // are reconstructed; `h`/`sv` come from storage. Look the config up by the + // resolved identifier so a nested JSON column (registered under a dotted + // path) is detected too. A v3 column builds the same `{ cast_as, indexes }` + // shape as a v2 one, so this detection needs no version branch. const columnConfig = encryptConfig.columns[toColumnName(matched)] if (columnConfig?.cast_as === 'json' && columnConfig.indexes.ste_vec) { + const stored = attrValue as { h?: unknown; sv?: unknown } return { [columnName]: { i, v, - // Mandatory in both versions — a v3 ste_vec document without it - // fails deserialization with "missing field `k`". k: 'sv', - sv: attrValue, + h: stored.h, + sv: stored.sv, }, } } diff --git a/packages/stack/src/dynamodb/types.ts b/packages/stack/src/dynamodb/types.ts index 5bcf8835..83dfa800 100644 --- a/packages/stack/src/dynamodb/types.ts +++ b/packages/stack/src/dynamodb/types.ts @@ -127,11 +127,15 @@ type V3Columns
= Table extends EncryptedV3Table ? C : never /** - * What `toEncryptedDynamoItem` writes into `__source` for a column: the - * `sv` array for a JSON document, the base64 ciphertext `c` for every scalar. + * What `toEncryptedDynamoItem` writes into `__source` for a column: for a + * JSON document, the SteVec entries plus the per-document KeyHeader `h` + * (`{ h, sv }`) — protect-ffi 0.30 decrypt requires `h`; for every scalar, the + * base64 ciphertext `c`. */ type SourceAttribute = - 'searchableJson' extends QueryTypesForColumn ? unknown[] : string + 'searchableJson' extends QueryTypesForColumn + ? { h: unknown; sv: unknown[] } + : string /** * Does this column mint the `hm` term that becomes `__hmac`? From ebddb2c8983f6fa559713b30525cb92c86414d4d Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 22 Jul 2026 10:19:21 +1000 Subject: [PATCH 16/20] fix(stack): close five DynamoDB v3 review-body findings with tests Findings raised in review bodies (never anchored, so untracked on GitHub), now fixed and covered: - Write path is column-aware: split only declared columns, matched on the same property path the read path rebuilds from (shared makeColumnMatcher). An undeclared nested payload is stored whole and round-trips instead of being split into a __source the read path could never reassemble. - Empty-string ciphertext: the scalar arm gates on presence ('c' in payload), so { v, i, c: '' } splits to __source: '' instead of falling through and leaking v/i as a raw map. - v2 nested __hmac drop via the bare-leaf fallback now has a pure test. - EncryptedAttributes dotted-path limitation pinned by a .test-d.ts assertion. - Arrays documented as a deliberate carve-out (code comments both paths, the EncryptedAttributes docblock, and the DynamoDB skill limitations). Coverage: real-payload array carve-out tests (write-whole, read-whole, round-trip) for v2 and v3, declared-column-holds-array, array-of-records, empty-ciphertext read round-trip, the v2 undeclared-nested twin, and two fast-check properties (array-nested envelope never split; write path never splits an undeclared payload). The shared ciphertext generator now includes the empty string. Non-vacuity proven by mutation (recursing arrays fails 8). 88 pure runtime tests, 85 type tests, 0 type errors, lint + build clean. --- .changeset/dynamodb-eql-v3.md | 10 ++ .../dynamodb/client-compat.test-d.ts | 27 +++++- .../__tests__/dynamodb/helpers-v3.test.ts | 94 +++++++++++++++++++ .../stack/__tests__/dynamodb/helpers.test.ts | 60 ++++++++++++ .../__tests__/dynamodb/properties.test.ts | 65 ++++++++++++- packages/stack/src/dynamodb/helpers.ts | 91 ++++++++++++------ .../operations/bulk-encrypt-models.ts | 5 +- .../src/dynamodb/operations/encrypt-model.ts | 7 +- packages/stack/src/dynamodb/types.ts | 6 ++ skills/stash-dynamodb/SKILL.md | 8 ++ 10 files changed, 339 insertions(+), 34 deletions(-) diff --git a/.changeset/dynamodb-eql-v3.md b/.changeset/dynamodb-eql-v3.md index d5eec1ef..c6303c99 100644 --- a/.changeset/dynamodb-eql-v3.md +++ b/.changeset/dynamodb-eql-v3.md @@ -52,6 +52,16 @@ Robustness, from review: - Caller input that cannot be structurally cloned no longer reaches the FFI by reference — the "encryption never mutates a caller's object" guarantee holds on that path too. +- The write path now splits only declared columns, matched on the same property + path the read path rebuilds from. A pre-encrypted payload placed under an + undeclared nested name is stored whole (and round-trips) instead of being + split into a `__source` the read path could never reassemble. +- A degenerate payload with an empty-string ciphertext is split like any other + ciphertext rather than falling through and being written as a raw map, which + had leaked its `v`/`i` envelope metadata into storage. +- Arrays are documented as a deliberate carve-out: the mapping does not descend + into them, so a payload inside a list is stored whole (still decryptable, but + not queryable and not part of the `__source`/`__hmac` layout). The v3 overloads are strongly typed. `encryptModel` / `bulkEncryptModels` check the input model against the table's column domains, and return the DynamoDB diff --git a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts index dc46d0c2..c5b83180 100644 --- a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts +++ b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts @@ -12,7 +12,7 @@ * awaiting yields a discriminated Result — are locked here, where `tsc` runs. */ import { describe, expectTypeOf, it } from 'vitest' -import type { EncryptedDynamoDBInstance } from '@/dynamodb' +import type { EncryptedAttributes, EncryptedDynamoDBInstance } from '@/dynamodb' import { encryptedDynamoDB } from '@/dynamodb' import type { EncryptionClient } from '@/encryption' import type { EncryptionV3 } from '@/encryption/v3' @@ -286,6 +286,31 @@ describe('decrypt passes through suffixed keys whose base names no column', () = }) }) +describe('a dotted-path v3 column is not modelled in EncryptedAttributes', () => { + it('passes the parent key through untyped and mints no split attribute', () => { + const dotted = encryptedTableV3('dotted_v3', { + 'profile.ssn': types.TextEq('profile.ssn'), + }) + type Model = { pk: string; profile: { ssn: string; name: string } } + type Mapped = EncryptedAttributes + + // LIMITATION pinned (see the `EncryptedAttributes` docblock in types.ts): a + // column declared under a dotted path is split INSIDE the nested `profile` + // map at runtime, but the model key is `profile`, not `profile.ssn`. The + // mapped type does not recognise `profile` as a column, so it passes through + // with its input type unchanged and no `__source`/`__hmac` term is + // minted at the type level. If a future change starts (or stops) modelling + // the nested split, this assertion breaks on purpose. + expectTypeOf().toEqualTypeOf<{ + ssn: string + name: string + }>() + expectTypeOf().not.toHaveProperty('profile.ssn__source') + expectTypeOf().not.toHaveProperty('profile__source') + expectTypeOf().toEqualTypeOf() + }) +}) + describe('the v2 overload still returns the input model', () => { it('keeps an existing v2 caller compiling unchanged', async () => { const result = await dynamo.encryptModel<{ pk: string; email?: string }>( diff --git a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts index 96a7552f..0c15574b 100644 --- a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts @@ -314,6 +314,41 @@ describe('regressions found in review', () => { }) }) + it('does not split a nested payload whose path names no declared column', () => { + // The write path must only split DECLARED columns. A pre-encrypted payload + // under an undeclared nested name used to be split into `__source` + // by shape alone — but the read path rebuilds only declared columns, so it + // would never reassemble it: silent, undecryptable data. Leave the whole + // envelope in place so write and read stay symmetric and it round-trips. + const item = { profile: { secret: scalar('secret', 'CT', { hm: 'H' }) } } + + const stored = toEncryptedDynamoItem(item, encryptedAttrs, true) + + expect(stored).toEqual(item) + expect(toItemWithEqlPayloads(stored, users)).toEqual(item) + }) + + it('splits a detected payload with an empty-string ciphertext rather than leaking its envelope', () => { + // `{ v, i, c: '' }` is DETECTED as a payload (it has v+i+c), but the scalar + // arm gated on truthiness (`if (payload.c)`), so an empty ciphertext fell + // through and was written out as a RAW MAP — leaking v/i into storage, the + // exact thing the "no envelope metadata leak" invariant forbids. Split it + // as `__source: ''` like any other ciphertext. + const stored = toEncryptedDynamoItem( + { email: { v: 3, i: { t: 'users', c: 'email' }, c: '' } }, + encryptedAttrs, + ) + + expect(stored).toEqual({ email__source: '' }) + + // The other half of the contract: read must rebuild `c: ''` intact — not + // drop it or treat the empty source as a missing attribute — or the value + // is unrecoverable. A truthiness regression on the read side would strip it. + expect(toItemWithEqlPayloads(stored, users)).toEqual({ + email: { i: { c: 'email', t: 'users' }, v: 3, c: '' }, + }) + }) + it('identifies a column by its DB name when it differs from the property', () => { // `emailAddress: types.TextEq('email_address')` — matching must happen on // the property name, identification on the DB name. @@ -343,6 +378,65 @@ describe('regressions found in review', () => { }) }) +describe('arrays are a deliberate carve-out', () => { + // The mapping descends into nested OBJECTS but never ARRAYS. A payload inside + // an array is therefore stored as its whole envelope, symmetrically skipped on + // read, so it still decrypts — it is just never split into a queryable + // `__source`/`__hmac`. These pin that carve-out with REAL payloads (the older + // "leaves arrays untouched" test used a `{ c }` lookalike the detector would + // skip anyway, so it did not characterise the skip). + + it('leaves a real payload inside an array whole rather than splitting it', () => { + const item = { tags: [scalar('tags', 'CT', { hm: 'H' })] } + + // Whole envelope retained (v, i, c, hm all present); no tags__source/__hmac. + expect(toEncryptedDynamoItem(item, encryptedAttrs, true)).toEqual(item) + }) + + it('passes an envelope nested in an array straight through on read', () => { + const item = { tags: [{ v: 3, i: { t: 'users', c: 'email' }, c: 'CT' }] } + + // Not recursed, not rebuilt, not dropped — the array element reaches the + // caller/FFI unchanged, which is what lets it decrypt. + expect(toItemWithEqlPayloads(item, users)).toEqual(item) + }) + + it('round-trips a payload nested in an array', () => { + const item = { tags: [scalar('tags', 'CT', { hm: 'H' })] } + + const stored = toEncryptedDynamoItem(item, encryptedAttrs, true) + expect(stored).toEqual(item) + expect(toItemWithEqlPayloads(stored, users)).toEqual(item) + }) + + it('does not split a declared column whose value is an array', () => { + // The one shape where the split branch's column gate DOES match + // (`matchColumn('email', '')` succeeds) — only `isStoredEqlPayload([...])` + // being false stops a split. That guard interaction is otherwise untested. + const item = { email: [scalar('email', 'CT', { hm: 'H' })] } + + const stored = toEncryptedDynamoItem(item, encryptedAttrs, true) + expect(stored).toEqual(item) + expect(stored).not.toHaveProperty('email__source') + expect(toItemWithEqlPayloads(stored, users)).toEqual(item) + }) + + it('leaves an array of objects each holding a payload whole, and round-trips it', () => { + // The realistic customer shape: a list of records, each with an encrypted + // field. Every element is stored and read whole. + const item = { + history: [ + { at: 'day1', email: scalar('email', 'CT1', { hm: 'H1' }) }, + { at: 'day2', email: scalar('email', 'CT2') }, + ], + } + + const stored = toEncryptedDynamoItem(item, encryptedAttrs, true) + expect(stored).toEqual(item) + expect(toItemWithEqlPayloads(stored, users)).toEqual(item) + }) +}) + describe('read context is resolved once per batch', () => { it('does not call table.build() per item when a context is passed', () => { // `buildReadContext` resolves the row-invariant facts once; the 3-arg form diff --git a/packages/stack/__tests__/dynamodb/helpers.test.ts b/packages/stack/__tests__/dynamodb/helpers.test.ts index 5a133093..37f4ef1c 100644 --- a/packages/stack/__tests__/dynamodb/helpers.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers.test.ts @@ -122,6 +122,30 @@ describe('toEncryptedDynamoItem (write path)', () => { expect(result).toEqual({ items: [{ c: 'looks-like-a-payload' }] }) }) + + it('leaves a real v2 payload inside an array whole and round-trips it', () => { + // The array carve-out with a genuine `v+i+c` payload (the case above uses a + // `{ c }` lookalike the detector skips anyway): stored whole, not split, and + // symmetrically passed through on read so it still decrypts. + const item = { tags: [ct('array-ct', 'array-hmac')] } + + const stored = toEncryptedDynamoItem(item, encryptedAttrs) + expect(stored).toEqual(item) + expect(stored).not.toHaveProperty('tags__source') + expect(toItemWithEqlPayloads(stored, users)).toEqual(item) + }) + + it('does not split a nested payload whose leaf names no declared v2 column', () => { + // The v2 branch carries the bare-leaf fallback; its POSITIVE case is tested + // above. The negative — a nested leaf that is NOT a declared column must be + // left whole (the read path rebuilds only declared columns, so a split here + // would be unrecoverable). Only the v3 branch (no fallback) covered this. + const item = { profile: { secret: ct('CT', 'H') } } + + const stored = toEncryptedDynamoItem(item, encryptedAttrs) + expect(stored).toEqual(item) + expect(toItemWithEqlPayloads(stored, users)).toEqual(item) + }) }) describe('toItemWithEqlPayloads (read path)', () => { @@ -207,6 +231,42 @@ describe('toItemWithEqlPayloads (read path)', () => { }) }) + it('splits and drops a nested v2 grouped field __hmac via the bare-leaf fallback', () => { + // `encryptedField('amount')` under a `details` group registers the bare + // leaf `amount`. A nested equality payload must split to `amount__hmac` on + // write and have it dropped on read through the v2 bare-leaf fallback — the + // v2 twin of the v3 dotted-path __hmac coverage, which was otherwise only + // exercised in the live suite. + const orders = encryptedTable('orders', { + details: { amount: encryptedField('amount') }, + }) + const attrs = Object.keys(orders.build().columns) + + const stored = toEncryptedDynamoItem( + { + details: { + amount: { + k: 'ct', + v: 2, + i: { t: 'orders', c: 'amount' }, + c: 'ct', + hm: 'h', + }, + }, + }, + attrs, + ) + expect(stored).toEqual({ + details: { amount__source: 'ct', amount__hmac: 'h' }, + }) + + expect(toItemWithEqlPayloads(stored, orders)).toEqual({ + details: { + amount: { i: { c: 'amount', t: 'orders' }, v: 2, k: 'ct', c: 'ct' }, + }, + }) + }) + it('passes plaintext attributes and null/undefined through untouched', () => { const item = { pk: 'user#1', role: 'admin', a: null, b: undefined } diff --git a/packages/stack/__tests__/dynamodb/properties.test.ts b/packages/stack/__tests__/dynamodb/properties.test.ts index a2873bac..f72ca706 100644 --- a/packages/stack/__tests__/dynamodb/properties.test.ts +++ b/packages/stack/__tests__/dynamodb/properties.test.ts @@ -70,8 +70,13 @@ const safeName = fc .stringMatching(/^[a-z][a-z0-9_]{0,7}$/) .filter((n) => !RESERVED_COLUMN_NAMES.has(n) && n !== PLAINTEXT_KEY) -/** Ciphertexts are never empty in practice, and the write path tests truthiness. */ -const ciphertext = fc.string({ minLength: 1 }) +/** + * Includes the empty string deliberately. The write path detects a scalar by + * the PRESENCE of `c`, not its truthiness, so `c: ''` must split to + * `__source: ''` and round-trip like any other ciphertext rather than + * falling through and leaking its `v`/`i` envelope into a stored raw map. + */ +const ciphertext = fc.string() /** ste_vec entries, as the FFI emits them for a JSON document. */ const steVecEntries = fc.array( @@ -553,3 +558,59 @@ describe('property: attributes are keyed by property name, identified by DB name ) }) }) + +// --------------------------------------------------------------------------- +// 8. ARRAY CARVE-OUT: the mapping descends into objects, never arrays +// --------------------------------------------------------------------------- + +describe('property: a payload inside an array is stored and read whole', () => { + // The write and read paths both skip arrays (`!Array.isArray`), so a real + // envelope wrapped in a list is stored as its whole self — never split into + // `__source`/`__hmac` — and passes back through unchanged, still + // decryptable. A regression that started recursing arrays would split the + // element and break this for any generated column/array-key combination. + it('never splits an array-nested envelope, in either direction, for any column set', () => { + fc.assert( + fc.property(safeName, safeName, ciphertext, (col, arrKey, ct) => { + const table = encryptedTable('t', { [col]: types.TextEq(col) }) + const attrs = Object.keys(table.buildColumnKeyMap()) + // A genuine payload (v+i+c+hm) wrapped in an array — under an arbitrary + // key, declared or not: arrays are skipped regardless. + const item = { + [arrKey]: [{ v: 3, i: { t: 't', c: col }, c: ct, hm: 'H' }], + } + + const stored = toEncryptedDynamoItem(item, attrs, true) + expect(stored).toEqual(item) + expect(toItemWithEqlPayloads(stored, table)).toEqual(item) + }), + ) + }) +}) + +// --------------------------------------------------------------------------- +// 9. COLUMN-AWARE WRITE GATE: split only declared columns +// --------------------------------------------------------------------------- + +describe('property: the write path never splits a payload naming no declared column', () => { + // The write path is column-aware, matched on the same property path the read + // path rebuilds from. A pre-encrypted payload placed under an UNDECLARED name + // — top-level or nested — must be left whole and round-trip, never split into + // a `__source` the read path (which rebuilds only declared columns) + // could never reassemble. + it('leaves an undeclared payload whole and round-trips it, top-level and nested', () => { + fc.assert( + fc.property(safeName, safeName, ciphertext, (col, other, ct) => { + fc.pre(col !== other) + const table = encryptedTable('t', { [col]: types.TextEq(col) }) + const attrs = Object.keys(table.buildColumnKeyMap()) + const payload = { v: 3, i: { t: 't', c: other }, c: ct, hm: 'H' } + const item = { [other]: payload, grp: { [other]: payload } } + + const stored = toEncryptedDynamoItem(item, attrs, true) + expect(stored).toEqual(item) + expect(toItemWithEqlPayloads(stored, table)).toEqual(item) + }), + ) + }) +}) diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index 66ee114c..55b99bdb 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -236,26 +236,56 @@ function isStoredEqlPayload(value: unknown): value is StoredEqlPayload { return 'c' in value || 'sv' in value } +/** + * Resolve an attribute (`leaf` under `prefix`) back to the column path it was + * declared under, or `undefined` if it names no declared column. Shared by the + * write and read paths so the two split and rebuild EXACTLY the same set of + * attributes — an asymmetry here writes data one side can never reassemble. + * + * `columnPaths` are the JS property paths a model's fields are matched on. The + * dotted form is tried first; a v2 grouped `encryptedField('amount')` registers + * the bare leaf, so a nested `amount` falls back to it. v3 always registers the + * full dotted path (`'profile.ssn'`), so it never needs the fallback — and must + * NOT use it, or a nested `note` would match a same-named TOP-LEVEL `note` + * column. Scope the fallback to nested v2 attributes only. + */ +export function makeColumnMatcher(isV3: boolean, columnPaths: string[]) { + return function matchColumn( + leaf: string, + prefix: string, + ): string | undefined { + const dotted = prefix ? `${prefix}.${leaf}` : leaf + if (columnPaths.includes(dotted)) return dotted + if (!isV3 && prefix && columnPaths.includes(leaf)) return leaf + return undefined + } +} + export function toEncryptedDynamoItem( encrypted: Record, encryptedAttrs: string[], + // `false` (v2) by default so existing 2-arg callers keep the v2 bare-leaf + // fallback; the operations pass `isV3Table(table)` so a v3 write splits the + // same columns a v3 read rebuilds. + isV3 = false, ): Record { + const matchColumn = makeColumnMatcher(isV3, encryptedAttrs) + function processValue( attrName: string, attrValue: unknown, - isNested: boolean, + prefix: string, ): Record { if (attrValue === null || attrValue === undefined) { return { [attrName]: attrValue } } - // Handle encrypted payload. Both arms require the value to actually BE a - // payload — a registered attribute name is not sufficient on its own, and - // for a nested value the name tells us nothing at all. - if ( - (encryptedAttrs.includes(attrName) || isNested) && - isStoredEqlPayload(attrValue) - ) { + // Handle encrypted payload. Split only a value that BOTH is a payload and + // names a declared column — matched on its property path, exactly as the + // read path rebuilds it. Splitting an undeclared nested payload (matched by + // shape alone) would write a `__source` the read path never + // reassembles, i.e. silent undecryptable data. + if (matchColumn(attrName, prefix) && isStoredEqlPayload(attrValue)) { const encryptPayload = attrValue // A JSON document keeps its `k: 'sv'` tag. Its index terms live *inside* @@ -276,8 +306,10 @@ export function toEncryptedDynamoItem( // Scalars. v2 tags every payload `k: 'ct'`; v3 scalars carry NO `k` // discriminator at all, so the presence of a ciphertext is the signal — // gating on `k === 'ct'` would drop every v3 scalar through to the - // nested-object branch below and write it out as a raw map. - if (encryptPayload?.c) { + // nested-object branch below and write it out as a raw map. Test + // PRESENCE, not truthiness: a `{ v, i, c: '' }` payload is still a + // payload, and letting it fall through would leak `v`/`i` into storage. + if ('c' in encryptPayload) { const result: Record = {} // `hm` is the deterministic equality term, and the only one a DynamoDB // key condition can use. Ordering terms (`op`/`ob`) and the match @@ -292,13 +324,25 @@ export function toEncryptedDynamoItem( } } - // Handle nested objects recursively + // Handle nested objects recursively, carrying the path so a nested column + // is matched against its registered dotted name. + // + // Arrays are a deliberate carve-out: they are NOT recursed into, so a + // payload inside an array is stored as its whole envelope rather than split + // into `__source`/`__hmac`. A DynamoDB key condition cannot + // target an array element anyway, so there is nothing to gain from a split; + // the read path skips arrays symmetrically, so such a value still + // round-trips and decrypts. Documented in the DynamoDB skill's limitations. if (typeof attrValue === 'object' && !Array.isArray(attrValue)) { const nestedResult = Object.entries( attrValue as Record, ).reduce( (acc, [key, val]) => { - const processed = processValue(key, val, true) + const processed = processValue( + key, + val, + prefix ? `${prefix}.${attrName}` : attrName, + ) Object.assign(acc, processed) return acc }, @@ -313,7 +357,7 @@ export function toEncryptedDynamoItem( return Object.entries(encrypted).reduce( (putItem, [attrName, attrValue]) => { - const processed = processValue(attrName, attrValue, false) + const processed = processValue(attrName, attrValue, '') Object.assign(putItem, processed) return putItem }, @@ -365,21 +409,8 @@ export function toItemWithEqlPayloads( ): Record { const { isV3, v, encryptConfig, columnPaths, toColumnName } = context - /** Resolve an attribute back to the column path it was written from. */ - function matchColumn(leaf: string, prefix: string): string | undefined { - const dotted = prefix ? `${prefix}.${leaf}` : leaf - if (columnPaths.includes(dotted)) return dotted - - // Bare-leaf fallback, v2-only. A v2 `encryptedField('amount')` inside a - // group registers the bare leaf `amount`, so a nested `amount__source` has - // to match it by leaf. v3 always registers the full dotted path - // (`'profile.ssn': types.TextEq('profile.ssn')`), so it never needs this — - // and must NOT use it: a nested `note__source` would otherwise match a - // same-named TOP-LEVEL `note` column and rewrite a plaintext sibling as an - // envelope. Scope the fallback to nested v2 attributes only. - if (!isV3 && prefix && columnPaths.includes(leaf)) return leaf - return undefined - } + // The same matcher the write path splits with, so the two stay symmetric. + const matchColumn = makeColumnMatcher(isV3, columnPaths) function processValue( attrName: string, @@ -463,7 +494,9 @@ export function toItemWithEqlPayloads( } // Handle nested objects recursively, carrying the path so a nested column - // can be matched against its registered dotted name. + // can be matched against its registered dotted name. Arrays are skipped + // symmetrically with the write path (which stores array-nested payloads + // whole), so such a value passes straight to the FFI and still decrypts. if (typeof attrValue === 'object' && !Array.isArray(attrValue)) { const nestedResult = Object.entries( attrValue as Record, diff --git a/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts b/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts index ff23b921..eafee15a 100644 --- a/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts +++ b/packages/stack/src/dynamodb/operations/bulk-encrypt-models.ts @@ -4,6 +4,7 @@ import { logger } from '@/utils/logger' import { deepClone, handleError, + isV3Table, throwPreservingCode, toEncryptedDynamoItem, } from '../helpers' @@ -63,8 +64,10 @@ export class BulkEncryptModelsOperation< this.table, ) + const isV3 = isV3Table(this.table) return data.map( - (encrypted) => toEncryptedDynamoItem(encrypted, encryptedAttrs) as T, + (encrypted) => + toEncryptedDynamoItem(encrypted, encryptedAttrs, isV3) as T, ) }, (error) => diff --git a/packages/stack/src/dynamodb/operations/encrypt-model.ts b/packages/stack/src/dynamodb/operations/encrypt-model.ts index 469988f4..56a56ea4 100644 --- a/packages/stack/src/dynamodb/operations/encrypt-model.ts +++ b/packages/stack/src/dynamodb/operations/encrypt-model.ts @@ -4,6 +4,7 @@ import { logger } from '@/utils/logger' import { deepClone, handleError, + isV3Table, throwPreservingCode, toEncryptedDynamoItem, } from '../helpers' @@ -60,7 +61,11 @@ export class EncryptModelOperation< this.table, ) - return toEncryptedDynamoItem(data, encryptedAttrs) as T + return toEncryptedDynamoItem( + data, + encryptedAttrs, + isV3Table(this.table), + ) as T }, (error) => handleError(error, 'encryptModel', { diff --git a/packages/stack/src/dynamodb/types.ts b/packages/stack/src/dynamodb/types.ts index 83dfa800..1bfbca98 100644 --- a/packages/stack/src/dynamodb/types.ts +++ b/packages/stack/src/dynamodb/types.ts @@ -180,6 +180,12 @@ type Simplify = { [K in keyof T]: T[K] } * split *inside* the nested `profile` map at runtime. The model key is * `profile`, not `profile.ssn`, so it passes through here unchanged and the * nested split is not modelled. + * + * LIMITATION: values inside an ARRAY are not descended into — the write path + * skips arrays, so a payload in a list is stored whole rather than split into + * `__source`/`__hmac`. It still decrypts on read, but this mapped type + * describes it as its plaintext input shape, not a split. Documented in the + * DynamoDB skill's limitations. */ export type EncryptedAttributes
= Simplify< { diff --git a/skills/stash-dynamodb/SKILL.md b/skills/stash-dynamodb/SKILL.md index fa952a1d..cf3d44d4 100644 --- a/skills/stash-dynamodb/SKILL.md +++ b/skills/stash-dynamodb/SKILL.md @@ -101,6 +101,14 @@ Both produce the same nested DynamoDB attribute *layout*, but the encrypted cont To encrypt a whole subtree as one value instead of per-leaf, use `types.Json`, which stores it as a single ste_vec attribute (`profile__source`). +> **Arrays are not descended into.** The adapter splits encrypted leaves inside +> nested *objects* only. A value inside an array is stored whole — it is not +> split into `__source`/`__hmac`, so its ciphertext still decrypts on read but +> can never back a key condition, and it does not appear in the `__source`/`__hmac` +> attribute layout above. A DynamoDB key condition cannot target an array element +> in any case. To encrypt list data, either promote the searched field to a +> top-level column, or wrap the subtree in a single `types.Json` column. + ## Rolling Encryption Out to Production DynamoDB encryption is **single-deploy**. There is no rollout/cutover split — unlike the Postgres path, DynamoDB has no row-level rename swap and no shared-state proxy. The application owns every write, so adding encryption is an application-side change that ships in one PR: From a4e6312d11d76ecf1e06b39f704ba2cb80bb2b3f Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 22 Jul 2026 11:17:13 +1000 Subject: [PATCH 17/20] test(stack): close two DynamoDB v3 audit gaps + harden test flakiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the PR #725 review-thread audit. Closes the two review requests that were resolved with a real gap versus what the reviewer asked, and hardens two flaky spots found during verification. Test-only; no production, public-API, or payload-shape change. - Thread 11: add the required-nullable type assertion to client-compat.test-d.ts — a required `string | null` column keeps `__source` required, while an optional column preserves `?`. The runtime tests existed; the requested type-level pin did not. Non-vacuity verified (required->optional fails test:types). - Thread 8: assert the 3-arg (prebuilt context) form of toItemWithEqlPayloads equals the 2-arg (default context) form — the equivalence the reviewer asked for alongside the build-once invariant. - Spy isolation: add afterEach(vi.restoreAllMocks()) to helpers-v3 and resolve-decrypt, which both spy the shared logger.debug singleton, so a spy can never bleed past a test boundary. - Live-suite flakiness: retry the live ZeroKMS v3 suites (retry: 2), scoped to those suites so pure/unit tests are never retried. --- .../dynamodb/client-compat.test-d.ts | 21 + .../dynamodb/encrypted-dynamodb-v3.test.ts | 642 ++++++++++-------- .../__tests__/dynamodb/helpers-v3.test.ts | 19 +- .../dynamodb/resolve-decrypt.test.ts | 10 +- 4 files changed, 390 insertions(+), 302 deletions(-) diff --git a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts index c5b83180..132abe5e 100644 --- a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts +++ b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts @@ -311,6 +311,27 @@ describe('a dotted-path v3 column is not modelled in EncryptedAttributes', () => }) }) +describe('a required-nullable v3 column keeps __source required', () => { + it('preserves the optional modifier only for optional columns', () => { + // `email` is required + nullable; `name` is optional. The `__source` half of + // `EncryptedAttributes` is a homomorphic key-remapped mapped type (types.ts), + // so it preserves the `?` modifier from the model key: a required-nullable + // column stays required, an optional column stays optional. The `| null` of + // the plaintext is dropped — `SourceAttribute` returns a flat `string` — so a + // null value writing no attribute at runtime is a deliberate, pinned mismatch + // with the optimistic type. `__hmac` is always optional regardless. + type Model = { pk: string; email: string | null; name?: string } + type Mapped = EncryptedAttributes + + expectTypeOf().toEqualTypeOf<{ + pk: string + email__source: string + email__hmac?: string + name__source?: string + }>() + }) +}) + describe('the v2 overload still returns the input model', () => { it('keeps an existing v2 caller compiling unchanged', async () => { const result = await dynamo.encryptModel<{ pk: string; email?: string }>( diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index d94d2314..f9e6d74b 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -39,6 +39,17 @@ const hasCipherStashCreds = [ 'CS_CLIENT_ACCESS_KEY', ].every((name) => Boolean(process.env[name])) +/** + * Every block below is a real ZeroKMS round trip. Under the default forks pool + * with file parallelism, many live suites hit ZeroKMS at once and a request can + * be transiently throttled — the test then reds even though the code is correct + * (it passes in isolation). Retry the live suites a couple of times so transient + * throttling doesn't flake CI. Scoped here rather than in `vitest.config.ts` so + * the pure/unit tests are never retried (a retry that hides a real pure-test bug + * is worse than the flake it would paper over). + */ +const liveSuiteOptions = { retry: 2 } as const + const users = encryptedTable('users_v3_dynamo', { email: types.TextEq('email'), name: types.Text('name'), @@ -81,264 +92,285 @@ beforeAll(async () => { nominalDynamo = encryptedDynamoDB({ encryptionClient: nominalClient }) }) -describe.skipIf(!hasCipherStashCreds)('encryptModel with a v3 table', () => { - it('splits an equality domain into __source and __hmac', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#1', email: 'alice@example.com', role: 'admin' }, - users, - ) - - if (result.failure) throw new Error(result.failure.message) - - expect(Object.keys(result.data).sort()).toEqual([ - 'email__hmac', - 'email__source', - 'pk', - 'role', - ]) - expect(typeof result.data.email__source).toBe('string') - expect(typeof result.data.email__hmac).toBe('string') - expect(result.data.email__source).not.toContain('alice@example.com') - }) +describe.skipIf(!hasCipherStashCreds)( + 'encryptModel with a v3 table', + liveSuiteOptions, + () => { + it('splits an equality domain into __source and __hmac', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#1', email: 'alice@example.com', role: 'admin' }, + users, + ) - it('regression: a v3 scalar is split, not written out as a raw map', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#2', name: 'Bob' }, - users, - ) + if (result.failure) throw new Error(result.failure.message) - if (result.failure) throw new Error(result.failure.message) + expect(Object.keys(result.data).sort()).toEqual([ + 'email__hmac', + 'email__source', + 'pk', + 'role', + ]) + expect(typeof result.data.email__source).toBe('string') + expect(typeof result.data.email__hmac).toBe('string') + expect(result.data.email__source).not.toContain('alice@example.com') + }) - // Before the port, gating on `k === 'ct'` meant untagged v3 scalars fell - // through to the nested-object branch and were stored as `{ name: { v, i, - // c } }` — no __source, and the ciphertext nested one level down. - expect(result.data).toHaveProperty('name__source') - expect(result.data).not.toHaveProperty('name') - expect(typeof result.data.name__source).toBe('string') - }) + it('regression: a v3 scalar is split, not written out as a raw map', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#2', name: 'Bob' }, + users, + ) - it('gives an ordering domain a __source but no __hmac', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#3', age: 42 }, - users, - ) + if (result.failure) throw new Error(result.failure.message) - if (result.failure) throw new Error(result.failure.message) + // Before the port, gating on `k === 'ct'` meant untagged v3 scalars fell + // through to the nested-object branch and were stored as `{ name: { v, i, + // c } }` — no __source, and the ciphertext nested one level down. + expect(result.data).toHaveProperty('name__source') + expect(result.data).not.toHaveProperty('name') + expect(typeof result.data.name__source).toBe('string') + }) - // `IntegerOrd` is equality-capable in Postgres via its ordering term, but - // that term has no DynamoDB query surface, so nothing is stored for it. - expect(result.data).toHaveProperty('age__source') - expect(result.data).not.toHaveProperty('age__hmac') - }) + it('gives an ordering domain a __source but no __hmac', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#3', age: 42 }, + users, + ) - it('keeps only the equality term of a TextSearch domain', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#4', bio: 'a long biography' }, - users, - ) + if (result.failure) throw new Error(result.failure.message) - if (result.failure) throw new Error(result.failure.message) + // `IntegerOrd` is equality-capable in Postgres via its ordering term, but + // that term has no DynamoDB query surface, so nothing is stored for it. + expect(result.data).toHaveProperty('age__source') + expect(result.data).not.toHaveProperty('age__hmac') + }) - // TextSearch mints hm + op + bf; only hm is storable as an attribute. - expect(Object.keys(result.data).sort()).toEqual([ - 'bio__hmac', - 'bio__source', - 'pk', - ]) - }) + it('keeps only the equality term of a TextSearch domain', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#4', bio: 'a long biography' }, + users, + ) - it('stores a Json domain as its sv entries plus the KeyHeader', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#5', meta: { a: 1, b: { c: 'deep' } } }, - users, - ) - - if (result.failure) throw new Error(result.failure.message) - - // A SteVec document is stored as `{ h, sv }` under __source: the `sv` - // entries plus the per-document KeyHeader `h` that protect-ffi 0.30 decrypt - // requires. `v`/`i`/`k` are reconstructed on read, so they are not stored. - const stored = result.data.meta__source as { h: unknown; sv: unknown[] } - expect(stored.h).toBeDefined() - expect(Array.isArray(stored.sv)).toBe(true) - expect(stored.sv.length).toBeGreaterThan(0) - }) + if (result.failure) throw new Error(result.failure.message) - it('passes a null column value through without splitting it', async () => { - // v3 reaches the null check through a different code path than v2: - // `isStoredEqlPayload` runs before the `k`/`c` gates, so a null must be - // recognised as a non-payload and passed through verbatim — no - // `email__source`, no `email__hmac` — with siblings intact. - const result = await typedDynamo.encryptModel( - { pk: 'user#null', email: null, role: 'admin' }, - users, - ) + // TextSearch mints hm + op + bf; only hm is storable as an attribute. + expect(Object.keys(result.data).sort()).toEqual([ + 'bio__hmac', + 'bio__source', + 'pk', + ]) + }) - if (result.failure) throw new Error(result.failure.message) + it('stores a Json domain as its sv entries plus the KeyHeader', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#5', meta: { a: 1, b: { c: 'deep' } } }, + users, + ) - expect(result.data.email).toBeNull() - expect(result.data).not.toHaveProperty('email__source') - expect(result.data).not.toHaveProperty('email__hmac') - expect(result.data.pk).toBe('user#null') - expect(result.data.role).toBe('admin') + if (result.failure) throw new Error(result.failure.message) - const decrypted = await typedDynamo.decryptModel(result.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + // A SteVec document is stored as `{ h, sv }` under __source: the `sv` + // entries plus the per-document KeyHeader `h` that protect-ffi 0.30 decrypt + // requires. `v`/`i`/`k` are reconstructed on read, so they are not stored. + const stored = result.data.meta__source as { h: unknown; sv: unknown[] } + expect(stored.h).toBeDefined() + expect(Array.isArray(stored.sv)).toBe(true) + expect(stored.sv.length).toBeGreaterThan(0) + }) - expect(decrypted.data).toEqual({ - pk: 'user#null', - email: null, - role: 'admin', + it('passes a null column value through without splitting it', async () => { + // v3 reaches the null check through a different code path than v2: + // `isStoredEqlPayload` runs before the `k`/`c` gates, so a null must be + // recognised as a non-payload and passed through verbatim — no + // `email__source`, no `email__hmac` — with siblings intact. + const result = await typedDynamo.encryptModel( + { pk: 'user#null', email: null, role: 'admin' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(result.data.email).toBeNull() + expect(result.data).not.toHaveProperty('email__source') + expect(result.data).not.toHaveProperty('email__hmac') + expect(result.data.pk).toBe('user#null') + expect(result.data.role).toBe('admin') + + const decrypted = await typedDynamo.decryptModel(result.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual({ + pk: 'user#null', + email: null, + role: 'admin', + }) }) - }) -}) + }, +) -describe.skipIf(!hasCipherStashCreds)('round trips with a v3 table', () => { - it('round-trips every domain back to the original item', async () => { - const original: User = { - pk: 'user#6', - email: 'erin@example.com', - name: 'Erin', - age: 42, - bio: 'a long biography', - meta: { a: 1, b: { c: 'deep' } }, - role: 'admin', - } +describe.skipIf(!hasCipherStashCreds)( + 'round trips with a v3 table', + liveSuiteOptions, + () => { + it('round-trips every domain back to the original item', async () => { + const original: User = { + pk: 'user#6', + email: 'erin@example.com', + name: 'Erin', + age: 42, + bio: 'a long biography', + meta: { a: 1, b: { c: 'deep' } }, + role: 'admin', + } - const encrypted = await typedDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await typedDynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await typedDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(original) - }) + expect(decrypted.data).toEqual(original) + }) - it('round-trips the same item through the nominal client', async () => { - const original: User = { - pk: 'user#7', - email: 'frank@example.com', - age: 7, - meta: { z: 1 }, - } + it('round-trips the same item through the nominal client', async () => { + const original: User = { + pk: 'user#7', + email: 'frank@example.com', + age: 7, + meta: { z: 1 }, + } - const encrypted = await nominalDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await nominalDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(original) - }) + expect(decrypted.data).toEqual(original) + }) - it('produces items either client can decrypt — the wire format is the same', async () => { - const original: User = { pk: 'user#8', email: 'grace@example.com' } + it('produces items either client can decrypt — the wire format is the same', async () => { + const original: User = { pk: 'user#8', email: 'grace@example.com' } - const encrypted = await typedDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(original) - }) + expect(decrypted.data).toEqual(original) + }) - /** - * The one LIVE property. Its pure siblings (`properties.test.ts`) cover the - * attribute mapping in both directions with a fake ciphertext; this closes the - * loop over the real thing — every run is a ZeroKMS encrypt + decrypt, so it - * is capped hard at 5 runs. It is the only check that the split/rebuild - * survives contact with actual ciphertext across all five domains at once. - */ - it('property: round-trips arbitrary multi-domain items through ZeroKMS', async () => { - const jsonLeaf = fc.oneof(fc.string(), fc.integer(), fc.boolean()) - const userArb = fc.record({ - pk: fc.string({ minLength: 1 }), - email: fc.string(), - name: fc.string(), - age: fc.integer(), - bio: fc.string(), - meta: fc.dictionary( - fc.string({ minLength: 1 }), - fc.oneof( - jsonLeaf, - fc.dictionary(fc.string({ minLength: 1 }), jsonLeaf, { maxKeys: 2 }), + /** + * The one LIVE property. Its pure siblings (`properties.test.ts`) cover the + * attribute mapping in both directions with a fake ciphertext; this closes the + * loop over the real thing — every run is a ZeroKMS encrypt + decrypt, so it + * is capped hard at 5 runs. It is the only check that the split/rebuild + * survives contact with actual ciphertext across all five domains at once. + */ + it('property: round-trips arbitrary multi-domain items through ZeroKMS', async () => { + const jsonLeaf = fc.oneof(fc.string(), fc.integer(), fc.boolean()) + const userArb = fc.record({ + pk: fc.string({ minLength: 1 }), + email: fc.string(), + name: fc.string(), + age: fc.integer(), + bio: fc.string(), + meta: fc.dictionary( + fc.string({ minLength: 1 }), + fc.oneof( + jsonLeaf, + fc.dictionary(fc.string({ minLength: 1 }), jsonLeaf, { + maxKeys: 2, + }), + ), + { minKeys: 1, maxKeys: 3 }, ), - { minKeys: 1, maxKeys: 3 }, - ), - role: fc.string(), - }) - - await fc.assert( - fc.asyncProperty(userArb, async (original) => { - const encrypted = await typedDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) - - const decrypted = await typedDynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) - - expect(decrypted.data).toEqual(original) - }), - // Deliberately small: each run is a real ZeroKMS round trip. The pure - // properties in `properties.test.ts` cover the attribute mapping - // exhaustively and for free; this one only needs to establish that the - // mapping composed with real encryption is identity-preserving over - // arbitrary items, which a handful of runs does. - { numRuns: 5 }, - ) - }, 120000) -}) + role: fc.string(), + }) -describe.skipIf(!hasCipherStashCreds)('bulk operations with a v3 table', () => { - it('encrypts and decrypts a batch', async () => { - const items: User[] = [ - { pk: 'user#9', email: 'a@example.com', name: 'A' }, - { pk: 'user#10', email: 'b@example.com', age: 3 }, - ] + await fc.assert( + fc.asyncProperty(userArb, async (original) => { + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await typedDynamo.decryptModel( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }), + // Deliberately small: each run is a real ZeroKMS round trip. The pure + // properties in `properties.test.ts` cover the attribute mapping + // exhaustively and for free; this one only needs to establish that the + // mapping composed with real encryption is identity-preserving over + // arbitrary items, which a handful of runs does. + { numRuns: 5 }, + ) + }, 120000) + }, +) - const encrypted = await typedDynamo.bulkEncryptModels(items, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) +describe.skipIf(!hasCipherStashCreds)( + 'bulk operations with a v3 table', + liveSuiteOptions, + () => { + it('encrypts and decrypts a batch', async () => { + const items: User[] = [ + { pk: 'user#9', email: 'a@example.com', name: 'A' }, + { pk: 'user#10', email: 'b@example.com', age: 3 }, + ] - expect(encrypted.data).toHaveLength(2) - for (const item of encrypted.data) { - expect(item).toHaveProperty('email__source') - expect(item).toHaveProperty('email__hmac') - } + const encrypted = await typedDynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await typedDynamo.bulkDecryptModels(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(encrypted.data).toHaveLength(2) + for (const item of encrypted.data) { + expect(item).toHaveProperty('email__source') + expect(item).toHaveProperty('email__hmac') + } - expect(decrypted.data).toEqual(items) - }) + const decrypted = await typedDynamo.bulkDecryptModels( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) - it('round-trips a batch through the nominal client', async () => { - const items: User[] = [ - { pk: 'user#11', email: 'c@example.com' }, - { pk: 'user#12', name: 'D' }, - ] + expect(decrypted.data).toEqual(items) + }) - const encrypted = await nominalDynamo.bulkEncryptModels(items, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + it('round-trips a batch through the nominal client', async () => { + const items: User[] = [ + { pk: 'user#11', email: 'c@example.com' }, + { pk: 'user#12', name: 'D' }, + ] - const decrypted = await nominalDynamo.bulkDecryptModels( - encrypted.data, - users, - ) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const encrypted = await nominalDynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - expect(decrypted.data).toEqual(items) - }) + const decrypted = await nominalDynamo.bulkDecryptModels( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) - it('accepts an empty batch', async () => { - const encrypted = await typedDynamo.bulkEncryptModels([], users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + expect(decrypted.data).toEqual(items) + }) - expect(encrypted.data).toEqual([]) - }) -}) + it('accepts an empty batch', async () => { + const encrypted = await typedDynamo.bulkEncryptModels([], users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + expect(encrypted.data).toEqual([]) + }) + }, +) describe.skipIf(!hasCipherStashCreds)( 'nested attributes with a v3 table', + liveSuiteOptions, () => { // EQL v3 has no nested-object *authoring* form — a nested group is a compile // error. But a dotted column path is a supported column name, and the model @@ -460,6 +492,7 @@ describe.skipIf(!hasCipherStashCreds)( describe.skipIf(!hasCipherStashCreds)( 'a v3 column whose property differs from its DB name', + liveSuiteOptions, () => { // Regression: `encryptedAttrs` was derived from `build().columns`, which for // v3 is keyed by DB name, while the encrypted model is keyed by property @@ -563,6 +596,7 @@ describe.skipIf(!hasCipherStashCreds)( describe.skipIf(!hasCipherStashCreds)( 'the __hmac key-condition path with a v3 table', + liveSuiteOptions, () => { it('mints, via encryptQuery, the same HMAC the item is stored under', async () => { const email = 'heidi@example.com' @@ -599,108 +633,116 @@ describe.skipIf(!hasCipherStashCreds)( }, ) -describe.skipIf(!hasCipherStashCreds)('audit metadata with a v3 table', () => { - const metadata = { sub: 'user-id-123', action: 'v3-port' } +describe.skipIf(!hasCipherStashCreds)( + 'audit metadata with a v3 table', + liveSuiteOptions, + () => { + const metadata = { sub: 'user-id-123', action: 'v3-port' } + + it('is carried on every operation of the nominal client', async () => { + const item: User = { pk: 'user#14', email: 'judy@example.com' } - it('is carried on every operation of the nominal client', async () => { - const item: User = { pk: 'user#14', email: 'judy@example.com' } + const encrypted = await nominalDynamo + .encryptModel(item, users) + .audit({ metadata }) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const encrypted = await nominalDynamo - .encryptModel(item, users) - .audit({ metadata }) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const decrypted = await nominalDynamo + .decryptModel(encrypted.data, users) + .audit({ metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) - const decrypted = await nominalDynamo - .decryptModel(encrypted.data, users) - .audit({ metadata }) - if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data).toEqual(item) + }) - expect(decrypted.data).toEqual(item) - }) + it('is accepted on the typed client, though decrypt cannot carry it', async () => { + const item: User = { pk: 'user#15', email: 'ken@example.com' } - it('is accepted on the typed client, though decrypt cannot carry it', async () => { - const item: User = { pk: 'user#15', email: 'ken@example.com' } + const encrypted = await typedDynamo + .encryptModel(item, users) + .audit({ metadata }) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const encrypted = await typedDynamo - .encryptModel(item, users) - .audit({ metadata }) - if (encrypted.failure) throw new Error(encrypted.failure.message) + // The typed client's `decryptModel` returns a plain promise with no audit + // surface. The chain must still resolve correctly — the metadata is simply + // not forwarded. Use the nominal client if decrypt audit matters. + const decrypted = await typedDynamo + .decryptModel(encrypted.data, users) + .audit({ metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) - // The typed client's `decryptModel` returns a plain promise with no audit - // surface. The chain must still resolve correctly — the metadata is simply - // not forwarded. Use the nominal client if decrypt audit matters. - const decrypted = await typedDynamo - .decryptModel(encrypted.data, users) - .audit({ metadata }) - if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data).toEqual(item) + }) + }, +) - expect(decrypted.data).toEqual(item) - }) -}) +describe.skipIf(!hasCipherStashCreds)( + 'error handling with a v3 table', + liveSuiteOptions, + () => { + const unknown = encryptedTable('users_v3_dynamo', { + nope: types.TextEq('nonexistent_column'), + }) -describe.skipIf(!hasCipherStashCreds)('error handling with a v3 table', () => { - const unknown = encryptedTable('users_v3_dynamo', { - nope: types.TextEq('nonexistent_column'), - }) + it('surfaces the FFI error code for an unregistered column', async () => { + const result = await typedDynamo.encryptModel({ nope: 'value' }, unknown) - it('surfaces the FFI error code for an unregistered column', async () => { - const result = await typedDynamo.encryptModel({ nope: 'value' }, unknown) + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) - expect(result.failure).toBeDefined() - expect(result.failure?.code).toBe('UNKNOWN_COLUMN') - }) + it('surfaces the FFI ciphertext error code for a malformed __source', async () => { + const result = await typedDynamo.decryptModel( + { email__source: 'not-a-ciphertext' }, + users, + ) - it('surfaces the FFI ciphertext error code for a malformed __source', async () => { - const result = await typedDynamo.decryptModel( - { email__source: 'not-a-ciphertext' }, - users, - ) - - expect(result.failure).toBeDefined() - expect(result.failure?.name).toBe('EncryptedDynamoDBError') - // Behaviour difference from v2, and an improvement: in v2 mode this path - // produced a bare "Unexpected end of input" with no code, so the adapter - // fell back to DYNAMODB_ENCRYPTION_ERROR (see the v2 suite). v3 rejects it - // as "Invalid EQL ciphertext" with a real FFI code, which is propagated. - expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') - expect(result.failure?.details).toEqual({ context: 'decryptModel' }) - }) + expect(result.failure).toBeDefined() + expect(result.failure?.name).toBe('EncryptedDynamoDBError') + // Behaviour difference from v2, and an improvement: in v2 mode this path + // produced a bare "Unexpected end of input" with no code, so the adapter + // fell back to DYNAMODB_ENCRYPTION_ERROR (see the v2 suite). v3 rejects it + // as "Invalid EQL ciphertext" with a real FFI code, which is propagated. + expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') + expect(result.failure?.details).toEqual({ context: 'decryptModel' }) + }) - it('surfaces the FFI error code on the bulk encrypt path', async () => { - const result = await typedDynamo.bulkEncryptModels( - [{ nope: 'value' }], - unknown, - ) + it('surfaces the FFI error code on the bulk encrypt path', async () => { + const result = await typedDynamo.bulkEncryptModels( + [{ nope: 'value' }], + unknown, + ) - expect(result.failure).toBeDefined() - expect(result.failure?.code).toBe('UNKNOWN_COLUMN') - }) + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) - it('surfaces the FFI ciphertext error code on the bulk decrypt path', async () => { - // The v3 bulk decrypt failure path (resolveDecryptResult + - // throwPreservingCode against a typed client) is otherwise never exercised - // end-to-end. Like the single-item v3 path, a malformed __source is rejected - // as "Invalid EQL ciphertext" with a real FFI code — not the v2 fallback. - const result = await typedDynamo.bulkDecryptModels( - [{ email__source: 'not-a-ciphertext' }], - users, - ) - - expect(result.failure).toBeDefined() - expect(result.failure?.name).toBe('EncryptedDynamoDBError') - expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') - expect(result.failure?.details).toEqual({ context: 'bulkDecryptModels' }) - }) + it('surfaces the FFI ciphertext error code on the bulk decrypt path', async () => { + // The v3 bulk decrypt failure path (resolveDecryptResult + + // throwPreservingCode against a typed client) is otherwise never exercised + // end-to-end. Like the single-item v3 path, a malformed __source is rejected + // as "Invalid EQL ciphertext" with a real FFI code — not the v2 fallback. + const result = await typedDynamo.bulkDecryptModels( + [{ email__source: 'not-a-ciphertext' }], + users, + ) - it('routes v3 failures to the configured errorHandler', async () => { - const seen: string[] = [] - const instrumented = encryptedDynamoDB({ - encryptionClient: nominalClient, - options: { errorHandler: (e) => seen.push(e.code) }, + expect(result.failure).toBeDefined() + expect(result.failure?.name).toBe('EncryptedDynamoDBError') + expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') + expect(result.failure?.details).toEqual({ context: 'bulkDecryptModels' }) }) - await instrumented.encryptModel({ nope: 'value' }, unknown) + it('routes v3 failures to the configured errorHandler', async () => { + const seen: string[] = [] + const instrumented = encryptedDynamoDB({ + encryptionClient: nominalClient, + options: { errorHandler: (e) => seen.push(e.code) }, + }) - expect(seen).toEqual(['UNKNOWN_COLUMN']) - }) -}) + await instrumented.encryptModel({ nope: 'value' }, unknown) + + expect(seen).toEqual(['UNKNOWN_COLUMN']) + }) + }, +) diff --git a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts index 0c15574b..543b2eb5 100644 --- a/packages/stack/__tests__/dynamodb/helpers-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/helpers-v3.test.ts @@ -13,7 +13,7 @@ * - a v3 JSON document keeps `k: 'sv'`, which is mandatory — deserialization * fails with "missing field `k`" without it. */ -import { describe, expect, it, vi } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { buildReadContext, deepClone, @@ -38,6 +38,16 @@ const users = encryptedTable('users', { const encryptedAttrs = Object.keys(users.build().columns) +// Several tests here `vi.spyOn(logger, 'debug')` — the same shared singleton that +// `resolve-decrypt.test.ts` also spies. Each test restores its own spy in a +// `finally`, but this hook is the safety net: it guarantees no spy on any shared +// object survives a test boundary, so a thrown assertion can never leak a patched +// method into the next test (`clearAllMocks` only clears calls; `restoreAllMocks` +// un-patches). +afterEach(() => { + vi.restoreAllMocks() +}) + /** A v3 scalar payload as the FFI returns it — note the absent `k`. */ const scalar = ( column: string, @@ -458,6 +468,13 @@ describe('read context is resolved once per batch', () => { expect(buildSpy).not.toHaveBeenCalled() + // ...and the 3-arg (prebuilt context) form produces the same item as the + // 2-arg (default context) form — passing a context is an optimisation, not a + // behaviour change. + expect(toItemWithEqlPayloads(items[0], users, context)).toEqual( + toItemWithEqlPayloads(items[0], users), + ) + buildSpy.mockRestore() }) }) diff --git a/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts b/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts index b1f74e09..f742afb9 100644 --- a/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts +++ b/packages/stack/__tests__/dynamodb/resolve-decrypt.test.ts @@ -5,10 +5,18 @@ * reachable through a live ZeroKMS decrypt; these move that assurance onto the * pure CI lane. No credentials, no network. */ -import { describe, expect, it, vi } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { resolveDecryptResult, throwPreservingCode } from '@/dynamodb/helpers' import { logger } from '@/utils/logger' +// The metadata-drop tests `vi.spyOn(logger, 'debug')` — the same shared singleton +// `helpers-v3.test.ts` also spies. Each test restores its own spy in a `finally`; +// this hook is the safety net so a patched method can never survive a test +// boundary (`restoreAllMocks` un-patches; `clearAllMocks` only clears calls). +afterEach(() => { + vi.restoreAllMocks() +}) + describe('resolveDecryptResult', () => { it('awaits a plain promise when the operation has no .audit (typed client)', async () => { const result = await resolveDecryptResult( From 57de9263d115f10e13cb2f6c7ebe86ebb1ee2649 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 22 Jul 2026 11:33:53 +1000 Subject: [PATCH 18/20] test(stack): fail client-compat type tests on Result failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback (PR #725): the type tests guarded the discriminated Result with `if (result.failure) return`, which silently passes the early-out branch instead of surfacing it. Match the repo's runtime dynamodb-test convention — `throw new Error(result.failure.message)` — so a failure is a hard failure, not a green no-op. Applied to all nine occurrences in the file, not only the two flagged, so the pattern is uniform. Type-narrowing is unchanged (throw terminates like return); test:types stays green. --- .../__tests__/dynamodb/client-compat.test-d.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts index 132abe5e..2c322357 100644 --- a/packages/stack/__tests__/dynamodb/client-compat.test-d.ts +++ b/packages/stack/__tests__/dynamodb/client-compat.test-d.ts @@ -139,7 +139,7 @@ describe('the v3 overload types the DynamoDB storage split', () => { { pk: 'a', email: 'a@b.com', age: 3, role: 'admin' }, usersV3, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) // Equality domain: ciphertext + the queryable HMAC term. expectTypeOf(result.data.email__source).toEqualTypeOf() @@ -159,7 +159,7 @@ describe('the v3 overload types the DynamoDB storage split', () => { { pk: 'a', meta: { a: 1 } }, usersV3, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) // A SteVec document is stored as `{ h, sv }` — the entries plus the // per-document KeyHeader `h` that protect-ffi 0.30 decrypt requires. @@ -175,7 +175,7 @@ describe('the v3 overload types the DynamoDB storage split', () => { [{ pk: 'a', email: 'a@b.com', age: 3 }], usersV3, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) expectTypeOf(result.data[0].email__source).toEqualTypeOf() expectTypeOf(result.data[0].email__hmac).toEqualTypeOf() @@ -188,7 +188,7 @@ describe('the v3 overload types the DynamoDB storage split', () => { [{ pk: 'a', email__source: 'ct', email__hmac: 'hm' }], usersV3, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) expectTypeOf(result.data[0].email).toEqualTypeOf() expectTypeOf(result.data[0]).not.toHaveProperty('email__source') @@ -206,7 +206,7 @@ describe('the v3 overload types the DynamoDB storage split', () => { { pk: 'a', email__source: 'ct', email__hmac: 'hm', role: 'admin' }, usersV3, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) expectTypeOf(result.data.email).toEqualTypeOf() // The query term is not data — it does not survive the read. @@ -219,7 +219,7 @@ describe('the v3 overload types the DynamoDB storage split', () => { it('round-trips a declared model shape', async () => { const model: V3Model = { pk: 'a', email: 'a@b.com', role: 'admin' } const encrypted = await dynamo.encryptModel(model, usersV3) - if (encrypted.failure) return + if (encrypted.failure) throw new Error(encrypted.failure.message) expectTypeOf(dynamo.decryptModel).toBeCallableWith(encrypted.data, usersV3) }) @@ -231,7 +231,7 @@ describe('a text-ordering domain reaches the HasSearchTerm `true` arm', () => { { pk: 'a', title: 'Hello', bio: 'about me' }, searchV3, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) // TextOrd is equality- AND order/range-capable over `string`: the innermost // `[PlaintextForColumn] extends [string] ? true : false` resolves `true`, @@ -265,7 +265,7 @@ describe('decrypt passes through suffixed keys whose base names no column', () = }, usersV3, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) // Declared column: `email__source` folds to `email`, `email__hmac` (its base // IS a column) is dropped as a query term. @@ -338,7 +338,7 @@ describe('the v2 overload still returns the input model', () => { { pk: 'a', email: 'a@b.com' }, usersV2, ) - if (result.failure) return + if (result.failure) throw new Error(result.failure.message) expectTypeOf(result.data).toEqualTypeOf<{ pk: string; email?: string }>() }) From 4161657cb3a4cb2a5910c89e2ca7b15255b4eb3e Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 22 Jul 2026 11:52:30 +1000 Subject: [PATCH 19/20] test(stack): fail the live v3 DynamoDB suite loudly on missing creds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback (PR #725): the `describe.skipIf(!hasCipherStashCreds)` gate turned absent credentials into a silent whole-suite skip on a green job — proving nothing and hiding regressions. This was the lone outlier: of the 23 live suites in packages/stack/__tests__, only this one gated; the other 22 (incl. the v2 dynamodb suite) already fail without creds. It also contradicted the test-kit contract, whose own header says "Integration suites FAIL when they are not configured. They never skip." Replace the skipIf gate with `requireIntegrationEnv(['cipherstash'])` in beforeAll — the purpose-built helper, which throws naming exactly which CS_* vars are missing (resolving from env vars OR a ~/.cipherstash profile). Drop the hasCipherStashCreds constant and the misleading docblock. The throttling `retry` is unchanged. Verified: green with creds; hard red (exit 1, not a skip) without them. --- .../dynamodb/encrypted-dynamodb-v3.test.ts | 856 +++++++++--------- 1 file changed, 410 insertions(+), 446 deletions(-) diff --git a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts index f9e6d74b..c32b41f1 100644 --- a/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts +++ b/packages/stack/__tests__/dynamodb/encrypted-dynamodb-v3.test.ts @@ -15,6 +15,7 @@ * Requires CipherStash credentials (`CS_*`), like the rest of this suite. */ import 'dotenv/config' +import { requireIntegrationEnv } from '@cipherstash/test-kit' import fc from 'fast-check' import { beforeAll, describe, expect, it } from 'vitest' import { encryptedDynamoDB } from '@/dynamodb' @@ -26,19 +27,6 @@ import type { JsonValue } from '@/eql/v3' import { encryptedTable, types } from '@/eql/v3' import { Encryption } from '@/index' -/** - * This suite talks to live ZeroKMS. Skip (rather than fail) when the CipherStash - * credentials are absent, so a local run without `.env` is green — matching the - * four variables `requireIntegrationEnv(['cipherstash'])` checks. With creds - * present the suite runs normally. - */ -const hasCipherStashCreds = [ - 'CS_WORKSPACE_CRN', - 'CS_CLIENT_ID', - 'CS_CLIENT_KEY', - 'CS_CLIENT_ACCESS_KEY', -].every((name) => Boolean(process.env[name])) - /** * Every block below is a real ZeroKMS round trip. Under the default forks pool * with file parallelism, many live suites hit ZeroKMS at once and a request can @@ -50,6 +38,10 @@ const hasCipherStashCreds = [ */ const liveSuiteOptions = { retry: 2 } as const +// Note: no `skipIf` gate — the suite requires credentials and fails without them +// (see `beforeAll`), matching the package convention. `liveSuiteOptions` only +// carries the throttling retry. + const users = encryptedTable('users_v3_dynamo', { email: types.TextEq('email'), name: types.Text('name'), @@ -78,9 +70,11 @@ let nominalClient: EncryptionClient let nominalDynamo: EncryptedDynamoDBInstance beforeAll(async () => { - // Nothing to set up when the suite is skipped for lack of credentials; the - // client constructors below would otherwise fail to authenticate. - if (!hasCipherStashCreds) return + // Fail loudly, never skip, when credentials are absent: a skipped live suite + // is a green run that proves nothing and hides regressions. `requireIntegrationEnv` + // throws naming exactly which of the four CS_* vars are missing — consistent + // with every other live suite in this package. + requireIntegrationEnv(['cipherstash']) const typedClient = await EncryptionV3({ schemas: [users] }) typedDynamo = encryptedDynamoDB({ encryptionClient: typedClient }) @@ -92,405 +86,383 @@ beforeAll(async () => { nominalDynamo = encryptedDynamoDB({ encryptionClient: nominalClient }) }) -describe.skipIf(!hasCipherStashCreds)( - 'encryptModel with a v3 table', - liveSuiteOptions, - () => { - it('splits an equality domain into __source and __hmac', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#1', email: 'alice@example.com', role: 'admin' }, - users, - ) - - if (result.failure) throw new Error(result.failure.message) - - expect(Object.keys(result.data).sort()).toEqual([ - 'email__hmac', - 'email__source', - 'pk', - 'role', - ]) - expect(typeof result.data.email__source).toBe('string') - expect(typeof result.data.email__hmac).toBe('string') - expect(result.data.email__source).not.toContain('alice@example.com') - }) - - it('regression: a v3 scalar is split, not written out as a raw map', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#2', name: 'Bob' }, - users, - ) - - if (result.failure) throw new Error(result.failure.message) +describe('encryptModel with a v3 table', liveSuiteOptions, () => { + it('splits an equality domain into __source and __hmac', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#1', email: 'alice@example.com', role: 'admin' }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + expect(Object.keys(result.data).sort()).toEqual([ + 'email__hmac', + 'email__source', + 'pk', + 'role', + ]) + expect(typeof result.data.email__source).toBe('string') + expect(typeof result.data.email__hmac).toBe('string') + expect(result.data.email__source).not.toContain('alice@example.com') + }) - // Before the port, gating on `k === 'ct'` meant untagged v3 scalars fell - // through to the nested-object branch and were stored as `{ name: { v, i, - // c } }` — no __source, and the ciphertext nested one level down. - expect(result.data).toHaveProperty('name__source') - expect(result.data).not.toHaveProperty('name') - expect(typeof result.data.name__source).toBe('string') - }) + it('regression: a v3 scalar is split, not written out as a raw map', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#2', name: 'Bob' }, + users, + ) - it('gives an ordering domain a __source but no __hmac', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#3', age: 42 }, - users, - ) + if (result.failure) throw new Error(result.failure.message) - if (result.failure) throw new Error(result.failure.message) + // Before the port, gating on `k === 'ct'` meant untagged v3 scalars fell + // through to the nested-object branch and were stored as `{ name: { v, i, + // c } }` — no __source, and the ciphertext nested one level down. + expect(result.data).toHaveProperty('name__source') + expect(result.data).not.toHaveProperty('name') + expect(typeof result.data.name__source).toBe('string') + }) - // `IntegerOrd` is equality-capable in Postgres via its ordering term, but - // that term has no DynamoDB query surface, so nothing is stored for it. - expect(result.data).toHaveProperty('age__source') - expect(result.data).not.toHaveProperty('age__hmac') - }) + it('gives an ordering domain a __source but no __hmac', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#3', age: 42 }, + users, + ) - it('keeps only the equality term of a TextSearch domain', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#4', bio: 'a long biography' }, - users, - ) + if (result.failure) throw new Error(result.failure.message) - if (result.failure) throw new Error(result.failure.message) + // `IntegerOrd` is equality-capable in Postgres via its ordering term, but + // that term has no DynamoDB query surface, so nothing is stored for it. + expect(result.data).toHaveProperty('age__source') + expect(result.data).not.toHaveProperty('age__hmac') + }) - // TextSearch mints hm + op + bf; only hm is storable as an attribute. - expect(Object.keys(result.data).sort()).toEqual([ - 'bio__hmac', - 'bio__source', - 'pk', - ]) - }) + it('keeps only the equality term of a TextSearch domain', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#4', bio: 'a long biography' }, + users, + ) - it('stores a Json domain as its sv entries plus the KeyHeader', async () => { - const result = await typedDynamo.encryptModel( - { pk: 'user#5', meta: { a: 1, b: { c: 'deep' } } }, - users, - ) + if (result.failure) throw new Error(result.failure.message) - if (result.failure) throw new Error(result.failure.message) + // TextSearch mints hm + op + bf; only hm is storable as an attribute. + expect(Object.keys(result.data).sort()).toEqual([ + 'bio__hmac', + 'bio__source', + 'pk', + ]) + }) - // A SteVec document is stored as `{ h, sv }` under __source: the `sv` - // entries plus the per-document KeyHeader `h` that protect-ffi 0.30 decrypt - // requires. `v`/`i`/`k` are reconstructed on read, so they are not stored. - const stored = result.data.meta__source as { h: unknown; sv: unknown[] } - expect(stored.h).toBeDefined() - expect(Array.isArray(stored.sv)).toBe(true) - expect(stored.sv.length).toBeGreaterThan(0) - }) + it('stores a Json domain as its sv entries plus the KeyHeader', async () => { + const result = await typedDynamo.encryptModel( + { pk: 'user#5', meta: { a: 1, b: { c: 'deep' } } }, + users, + ) + + if (result.failure) throw new Error(result.failure.message) + + // A SteVec document is stored as `{ h, sv }` under __source: the `sv` + // entries plus the per-document KeyHeader `h` that protect-ffi 0.30 decrypt + // requires. `v`/`i`/`k` are reconstructed on read, so they are not stored. + const stored = result.data.meta__source as { h: unknown; sv: unknown[] } + expect(stored.h).toBeDefined() + expect(Array.isArray(stored.sv)).toBe(true) + expect(stored.sv.length).toBeGreaterThan(0) + }) - it('passes a null column value through without splitting it', async () => { - // v3 reaches the null check through a different code path than v2: - // `isStoredEqlPayload` runs before the `k`/`c` gates, so a null must be - // recognised as a non-payload and passed through verbatim — no - // `email__source`, no `email__hmac` — with siblings intact. - const result = await typedDynamo.encryptModel( - { pk: 'user#null', email: null, role: 'admin' }, - users, - ) + it('passes a null column value through without splitting it', async () => { + // v3 reaches the null check through a different code path than v2: + // `isStoredEqlPayload` runs before the `k`/`c` gates, so a null must be + // recognised as a non-payload and passed through verbatim — no + // `email__source`, no `email__hmac` — with siblings intact. + const result = await typedDynamo.encryptModel( + { pk: 'user#null', email: null, role: 'admin' }, + users, + ) - if (result.failure) throw new Error(result.failure.message) + if (result.failure) throw new Error(result.failure.message) - expect(result.data.email).toBeNull() - expect(result.data).not.toHaveProperty('email__source') - expect(result.data).not.toHaveProperty('email__hmac') - expect(result.data.pk).toBe('user#null') - expect(result.data.role).toBe('admin') + expect(result.data.email).toBeNull() + expect(result.data).not.toHaveProperty('email__source') + expect(result.data).not.toHaveProperty('email__hmac') + expect(result.data.pk).toBe('user#null') + expect(result.data.role).toBe('admin') - const decrypted = await typedDynamo.decryptModel(result.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await typedDynamo.decryptModel(result.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual({ - pk: 'user#null', - email: null, - role: 'admin', - }) + expect(decrypted.data).toEqual({ + pk: 'user#null', + email: null, + role: 'admin', }) - }, -) + }) +}) -describe.skipIf(!hasCipherStashCreds)( - 'round trips with a v3 table', - liveSuiteOptions, - () => { - it('round-trips every domain back to the original item', async () => { - const original: User = { - pk: 'user#6', - email: 'erin@example.com', - name: 'Erin', - age: 42, - bio: 'a long biography', - meta: { a: 1, b: { c: 'deep' } }, - role: 'admin', - } +describe('round trips with a v3 table', liveSuiteOptions, () => { + it('round-trips every domain back to the original item', async () => { + const original: User = { + pk: 'user#6', + email: 'erin@example.com', + name: 'Erin', + age: 42, + bio: 'a long biography', + meta: { a: 1, b: { c: 'deep' } }, + role: 'admin', + } - const encrypted = await typedDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await typedDynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await typedDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(original) - }) + expect(decrypted.data).toEqual(original) + }) - it('round-trips the same item through the nominal client', async () => { - const original: User = { - pk: 'user#7', - email: 'frank@example.com', - age: 7, - meta: { z: 1 }, - } + it('round-trips the same item through the nominal client', async () => { + const original: User = { + pk: 'user#7', + email: 'frank@example.com', + age: 7, + meta: { z: 1 }, + } - const encrypted = await nominalDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await nominalDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(original) - }) + expect(decrypted.data).toEqual(original) + }) - it('produces items either client can decrypt — the wire format is the same', async () => { - const original: User = { pk: 'user#8', email: 'grace@example.com' } + it('produces items either client can decrypt — the wire format is the same', async () => { + const original: User = { pk: 'user#8', email: 'grace@example.com' } - const encrypted = await typedDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await nominalDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(original) - }) + expect(decrypted.data).toEqual(original) + }) - /** - * The one LIVE property. Its pure siblings (`properties.test.ts`) cover the - * attribute mapping in both directions with a fake ciphertext; this closes the - * loop over the real thing — every run is a ZeroKMS encrypt + decrypt, so it - * is capped hard at 5 runs. It is the only check that the split/rebuild - * survives contact with actual ciphertext across all five domains at once. - */ - it('property: round-trips arbitrary multi-domain items through ZeroKMS', async () => { - const jsonLeaf = fc.oneof(fc.string(), fc.integer(), fc.boolean()) - const userArb = fc.record({ - pk: fc.string({ minLength: 1 }), - email: fc.string(), - name: fc.string(), - age: fc.integer(), - bio: fc.string(), - meta: fc.dictionary( - fc.string({ minLength: 1 }), - fc.oneof( - jsonLeaf, - fc.dictionary(fc.string({ minLength: 1 }), jsonLeaf, { - maxKeys: 2, - }), - ), - { minKeys: 1, maxKeys: 3 }, + /** + * The one LIVE property. Its pure siblings (`properties.test.ts`) cover the + * attribute mapping in both directions with a fake ciphertext; this closes the + * loop over the real thing — every run is a ZeroKMS encrypt + decrypt, so it + * is capped hard at 5 runs. It is the only check that the split/rebuild + * survives contact with actual ciphertext across all five domains at once. + */ + it('property: round-trips arbitrary multi-domain items through ZeroKMS', async () => { + const jsonLeaf = fc.oneof(fc.string(), fc.integer(), fc.boolean()) + const userArb = fc.record({ + pk: fc.string({ minLength: 1 }), + email: fc.string(), + name: fc.string(), + age: fc.integer(), + bio: fc.string(), + meta: fc.dictionary( + fc.string({ minLength: 1 }), + fc.oneof( + jsonLeaf, + fc.dictionary(fc.string({ minLength: 1 }), jsonLeaf, { + maxKeys: 2, + }), ), - role: fc.string(), - }) - - await fc.assert( - fc.asyncProperty(userArb, async (original) => { - const encrypted = await typedDynamo.encryptModel(original, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) - - const decrypted = await typedDynamo.decryptModel( - encrypted.data, - users, - ) - if (decrypted.failure) throw new Error(decrypted.failure.message) - - expect(decrypted.data).toEqual(original) - }), - // Deliberately small: each run is a real ZeroKMS round trip. The pure - // properties in `properties.test.ts` cover the attribute mapping - // exhaustively and for free; this one only needs to establish that the - // mapping composed with real encryption is identity-preserving over - // arbitrary items, which a handful of runs does. - { numRuns: 5 }, - ) - }, 120000) - }, -) - -describe.skipIf(!hasCipherStashCreds)( - 'bulk operations with a v3 table', - liveSuiteOptions, - () => { - it('encrypts and decrypts a batch', async () => { - const items: User[] = [ - { pk: 'user#9', email: 'a@example.com', name: 'A' }, - { pk: 'user#10', email: 'b@example.com', age: 3 }, - ] - - const encrypted = await typedDynamo.bulkEncryptModels(items, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) - - expect(encrypted.data).toHaveLength(2) - for (const item of encrypted.data) { - expect(item).toHaveProperty('email__source') - expect(item).toHaveProperty('email__hmac') - } + { minKeys: 1, maxKeys: 3 }, + ), + role: fc.string(), + }) + + await fc.assert( + fc.asyncProperty(userArb, async (original) => { + const encrypted = await typedDynamo.encryptModel(original, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const decrypted = await typedDynamo.decryptModel(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + expect(decrypted.data).toEqual(original) + }), + // Deliberately small: each run is a real ZeroKMS round trip. The pure + // properties in `properties.test.ts` cover the attribute mapping + // exhaustively and for free; this one only needs to establish that the + // mapping composed with real encryption is identity-preserving over + // arbitrary items, which a handful of runs does. + { numRuns: 5 }, + ) + }, 120000) +}) - const decrypted = await typedDynamo.bulkDecryptModels( - encrypted.data, - users, - ) - if (decrypted.failure) throw new Error(decrypted.failure.message) +describe('bulk operations with a v3 table', liveSuiteOptions, () => { + it('encrypts and decrypts a batch', async () => { + const items: User[] = [ + { pk: 'user#9', email: 'a@example.com', name: 'A' }, + { pk: 'user#10', email: 'b@example.com', age: 3 }, + ] - expect(decrypted.data).toEqual(items) - }) + const encrypted = await typedDynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - it('round-trips a batch through the nominal client', async () => { - const items: User[] = [ - { pk: 'user#11', email: 'c@example.com' }, - { pk: 'user#12', name: 'D' }, - ] + expect(encrypted.data).toHaveLength(2) + for (const item of encrypted.data) { + expect(item).toHaveProperty('email__source') + expect(item).toHaveProperty('email__hmac') + } - const encrypted = await nominalDynamo.bulkEncryptModels(items, users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const decrypted = await typedDynamo.bulkDecryptModels(encrypted.data, users) + if (decrypted.failure) throw new Error(decrypted.failure.message) - const decrypted = await nominalDynamo.bulkDecryptModels( - encrypted.data, - users, - ) - if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data).toEqual(items) + }) - expect(decrypted.data).toEqual(items) - }) + it('round-trips a batch through the nominal client', async () => { + const items: User[] = [ + { pk: 'user#11', email: 'c@example.com' }, + { pk: 'user#12', name: 'D' }, + ] - it('accepts an empty batch', async () => { - const encrypted = await typedDynamo.bulkEncryptModels([], users) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await nominalDynamo.bulkEncryptModels(items, users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - expect(encrypted.data).toEqual([]) - }) - }, -) + const decrypted = await nominalDynamo.bulkDecryptModels( + encrypted.data, + users, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) -describe.skipIf(!hasCipherStashCreds)( - 'nested attributes with a v3 table', - liveSuiteOptions, - () => { - // EQL v3 has no nested-object *authoring* form — a nested group is a compile - // error. But a dotted column path is a supported column name, and the model - // walk in `encryption/helpers/model-helpers.ts` is shared with v2 and matches - // on dotted paths. So nested DynamoDB attributes work in v3 today, declared - // flat. This matters for DynamoDB specifically, where items are natively - // nested documents. - const nested = encryptedTable('users_v3_nested', { - 'profile.ssn': types.TextEq('profile.ssn'), - 'profile.note': types.Text('profile.note'), - }) + expect(decrypted.data).toEqual(items) + }) - type NestedUser = { - pk: string - profile?: { ssn?: string; note?: string; city?: string } - } + it('accepts an empty batch', async () => { + const encrypted = await typedDynamo.bulkEncryptModels([], users) + if (encrypted.failure) throw new Error(encrypted.failure.message) - let nestedDynamo: EncryptedDynamoDBInstance - let nestedClient: EncryptionClient + expect(encrypted.data).toEqual([]) + }) +}) - beforeAll(async () => { - nestedClient = await Encryption({ - schemas: [nested] as never, - config: { eqlVersion: 3 }, - }) - nestedDynamo = encryptedDynamoDB({ encryptionClient: nestedClient }) - }) +describe('nested attributes with a v3 table', liveSuiteOptions, () => { + // EQL v3 has no nested-object *authoring* form — a nested group is a compile + // error. But a dotted column path is a supported column name, and the model + // walk in `encryption/helpers/model-helpers.ts` is shared with v2 and matches + // on dotted paths. So nested DynamoDB attributes work in v3 today, declared + // flat. This matters for DynamoDB specifically, where items are natively + // nested documents. + const nested = encryptedTable('users_v3_nested', { + 'profile.ssn': types.TextEq('profile.ssn'), + 'profile.note': types.Text('profile.note'), + }) - it('splits nested attributes in place, leaving plaintext siblings alone', async () => { - const result = await nestedDynamo.encryptModel( - { - pk: 'u#1', - profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, - }, - nested, - ) + type NestedUser = { + pk: string + profile?: { ssn?: string; note?: string; city?: string } + } - if (result.failure) throw new Error(result.failure.message) + let nestedDynamo: EncryptedDynamoDBInstance + let nestedClient: EncryptionClient - const profile = result.data.profile as Record - expect(Object.keys(profile).sort()).toEqual([ - 'city', - 'note__source', - 'ssn__hmac', - 'ssn__source', - ]) - expect(profile.city).toBe('Sydney') + beforeAll(async () => { + nestedClient = await Encryption({ + schemas: [nested] as never, + config: { eqlVersion: 3 }, }) + nestedDynamo = encryptedDynamoDB({ encryptionClient: nestedClient }) + }) - it('round-trips a nested item exactly', async () => { - const original: NestedUser = { - pk: 'u#2', + it('splits nested attributes in place, leaving plaintext siblings alone', async () => { + const result = await nestedDynamo.encryptModel( + { + pk: 'u#1', profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, - } + }, + nested, + ) + + if (result.failure) throw new Error(result.failure.message) + + const profile = result.data.profile as Record + expect(Object.keys(profile).sort()).toEqual([ + 'city', + 'note__source', + 'ssn__hmac', + 'ssn__source', + ]) + expect(profile.city).toBe('Sydney') + }) - const encrypted = await nestedDynamo.encryptModel(original, nested) - if (encrypted.failure) throw new Error(encrypted.failure.message) + it('round-trips a nested item exactly', async () => { + const original: NestedUser = { + pk: 'u#2', + profile: { ssn: '123-45-6789', note: 'hi', city: 'Sydney' }, + } - const decrypted = await nestedDynamo.decryptModel(encrypted.data, nested) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const encrypted = await nestedDynamo.encryptModel(original, nested) + if (encrypted.failure) throw new Error(encrypted.failure.message) - expect(decrypted.data).toEqual(original) - }) + const decrypted = await nestedDynamo.decryptModel(encrypted.data, nested) + if (decrypted.failure) throw new Error(decrypted.failure.message) - it('rebuilds the nested identifier from its registered dotted name', async () => { - const encrypted = await nestedDynamo.encryptModel( - { pk: 'u#3', profile: { ssn: '123-45-6789' } }, - nested, - ) - if (encrypted.failure) throw new Error(encrypted.failure.message) + expect(decrypted.data).toEqual(original) + }) - const rebuilt = toItemWithEqlPayloads(encrypted.data, nested) - const payload = (rebuilt.profile as Record) - .ssn + it('rebuilds the nested identifier from its registered dotted name', async () => { + const encrypted = await nestedDynamo.encryptModel( + { pk: 'u#3', profile: { ssn: '123-45-6789' } }, + nested, + ) + if (encrypted.failure) throw new Error(encrypted.failure.message) - expect(payload.i.c).toBe('profile.ssn') - }) + const rebuilt = toItemWithEqlPayloads(encrypted.data, nested) + const payload = (rebuilt.profile as Record) + .ssn - it('makes a nested equality attribute queryable by __hmac', async () => { - const ssn = '999-88-7777' + expect(payload.i.c).toBe('profile.ssn') + }) - const stored = await nestedDynamo.encryptModel( - { pk: 'u#4', profile: { ssn } }, - nested, - ) - if (stored.failure) throw new Error(stored.failure.message) + it('makes a nested equality attribute queryable by __hmac', async () => { + const ssn = '999-88-7777' - const term = await nestedClient.encryptQuery(ssn, { - table: nested as never, - column: nested['profile.ssn'] as never, - }) - if (term.failure) throw new Error(term.failure.message) + const stored = await nestedDynamo.encryptModel( + { pk: 'u#4', profile: { ssn } }, + nested, + ) + if (stored.failure) throw new Error(stored.failure.message) - const profile = stored.data.profile as Record - // The query term matches the stored nested `profile.ssn__hmac` — usable in a - // FilterExpression (a nested attribute can't back a DynamoDB key condition). - expect((term.data as { hm: string }).hm).toBe(profile.ssn__hmac) + const term = await nestedClient.encryptQuery(ssn, { + table: nested as never, + column: nested['profile.ssn'] as never, }) + if (term.failure) throw new Error(term.failure.message) - it('bulk round-trips nested items', async () => { - const items: NestedUser[] = [ - { pk: 'u#5', profile: { ssn: 'a', city: 'Sydney' } }, - { pk: 'u#6', profile: { note: 'b' } }, - ] + const profile = stored.data.profile as Record + // The query term matches the stored nested `profile.ssn__hmac` — usable in a + // FilterExpression (a nested attribute can't back a DynamoDB key condition). + expect((term.data as { hm: string }).hm).toBe(profile.ssn__hmac) + }) - const encrypted = await nestedDynamo.bulkEncryptModels(items, nested) - if (encrypted.failure) throw new Error(encrypted.failure.message) + it('bulk round-trips nested items', async () => { + const items: NestedUser[] = [ + { pk: 'u#5', profile: { ssn: 'a', city: 'Sydney' } }, + { pk: 'u#6', profile: { note: 'b' } }, + ] - const decrypted = await nestedDynamo.bulkDecryptModels( - encrypted.data, - nested, - ) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const encrypted = await nestedDynamo.bulkEncryptModels(items, nested) + if (encrypted.failure) throw new Error(encrypted.failure.message) - expect(decrypted.data).toEqual(items) - }) - }, -) + const decrypted = await nestedDynamo.bulkDecryptModels( + encrypted.data, + nested, + ) + if (decrypted.failure) throw new Error(decrypted.failure.message) -describe.skipIf(!hasCipherStashCreds)( + expect(decrypted.data).toEqual(items) + }) +}) + +describe( 'a v3 column whose property differs from its DB name', liveSuiteOptions, () => { @@ -594,7 +566,7 @@ describe.skipIf(!hasCipherStashCreds)( }, ) -describe.skipIf(!hasCipherStashCreds)( +describe( 'the __hmac key-condition path with a v3 table', liveSuiteOptions, () => { @@ -633,116 +605,108 @@ describe.skipIf(!hasCipherStashCreds)( }, ) -describe.skipIf(!hasCipherStashCreds)( - 'audit metadata with a v3 table', - liveSuiteOptions, - () => { - const metadata = { sub: 'user-id-123', action: 'v3-port' } +describe('audit metadata with a v3 table', liveSuiteOptions, () => { + const metadata = { sub: 'user-id-123', action: 'v3-port' } - it('is carried on every operation of the nominal client', async () => { - const item: User = { pk: 'user#14', email: 'judy@example.com' } + it('is carried on every operation of the nominal client', async () => { + const item: User = { pk: 'user#14', email: 'judy@example.com' } - const encrypted = await nominalDynamo - .encryptModel(item, users) - .audit({ metadata }) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await nominalDynamo + .encryptModel(item, users) + .audit({ metadata }) + if (encrypted.failure) throw new Error(encrypted.failure.message) - const decrypted = await nominalDynamo - .decryptModel(encrypted.data, users) - .audit({ metadata }) - if (decrypted.failure) throw new Error(decrypted.failure.message) + const decrypted = await nominalDynamo + .decryptModel(encrypted.data, users) + .audit({ metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(item) - }) + expect(decrypted.data).toEqual(item) + }) - it('is accepted on the typed client, though decrypt cannot carry it', async () => { - const item: User = { pk: 'user#15', email: 'ken@example.com' } + it('is accepted on the typed client, though decrypt cannot carry it', async () => { + const item: User = { pk: 'user#15', email: 'ken@example.com' } - const encrypted = await typedDynamo - .encryptModel(item, users) - .audit({ metadata }) - if (encrypted.failure) throw new Error(encrypted.failure.message) + const encrypted = await typedDynamo + .encryptModel(item, users) + .audit({ metadata }) + if (encrypted.failure) throw new Error(encrypted.failure.message) - // The typed client's `decryptModel` returns a plain promise with no audit - // surface. The chain must still resolve correctly — the metadata is simply - // not forwarded. Use the nominal client if decrypt audit matters. - const decrypted = await typedDynamo - .decryptModel(encrypted.data, users) - .audit({ metadata }) - if (decrypted.failure) throw new Error(decrypted.failure.message) + // The typed client's `decryptModel` returns a plain promise with no audit + // surface. The chain must still resolve correctly — the metadata is simply + // not forwarded. Use the nominal client if decrypt audit matters. + const decrypted = await typedDynamo + .decryptModel(encrypted.data, users) + .audit({ metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) - expect(decrypted.data).toEqual(item) - }) - }, -) - -describe.skipIf(!hasCipherStashCreds)( - 'error handling with a v3 table', - liveSuiteOptions, - () => { - const unknown = encryptedTable('users_v3_dynamo', { - nope: types.TextEq('nonexistent_column'), - }) + expect(decrypted.data).toEqual(item) + }) +}) - it('surfaces the FFI error code for an unregistered column', async () => { - const result = await typedDynamo.encryptModel({ nope: 'value' }, unknown) +describe('error handling with a v3 table', liveSuiteOptions, () => { + const unknown = encryptedTable('users_v3_dynamo', { + nope: types.TextEq('nonexistent_column'), + }) - expect(result.failure).toBeDefined() - expect(result.failure?.code).toBe('UNKNOWN_COLUMN') - }) + it('surfaces the FFI error code for an unregistered column', async () => { + const result = await typedDynamo.encryptModel({ nope: 'value' }, unknown) - it('surfaces the FFI ciphertext error code for a malformed __source', async () => { - const result = await typedDynamo.decryptModel( - { email__source: 'not-a-ciphertext' }, - users, - ) + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) - expect(result.failure).toBeDefined() - expect(result.failure?.name).toBe('EncryptedDynamoDBError') - // Behaviour difference from v2, and an improvement: in v2 mode this path - // produced a bare "Unexpected end of input" with no code, so the adapter - // fell back to DYNAMODB_ENCRYPTION_ERROR (see the v2 suite). v3 rejects it - // as "Invalid EQL ciphertext" with a real FFI code, which is propagated. - expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') - expect(result.failure?.details).toEqual({ context: 'decryptModel' }) - }) + it('surfaces the FFI ciphertext error code for a malformed __source', async () => { + const result = await typedDynamo.decryptModel( + { email__source: 'not-a-ciphertext' }, + users, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.name).toBe('EncryptedDynamoDBError') + // Behaviour difference from v2, and an improvement: in v2 mode this path + // produced a bare "Unexpected end of input" with no code, so the adapter + // fell back to DYNAMODB_ENCRYPTION_ERROR (see the v2 suite). v3 rejects it + // as "Invalid EQL ciphertext" with a real FFI code, which is propagated. + expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') + expect(result.failure?.details).toEqual({ context: 'decryptModel' }) + }) - it('surfaces the FFI error code on the bulk encrypt path', async () => { - const result = await typedDynamo.bulkEncryptModels( - [{ nope: 'value' }], - unknown, - ) + it('surfaces the FFI error code on the bulk encrypt path', async () => { + const result = await typedDynamo.bulkEncryptModels( + [{ nope: 'value' }], + unknown, + ) - expect(result.failure).toBeDefined() - expect(result.failure?.code).toBe('UNKNOWN_COLUMN') - }) + expect(result.failure).toBeDefined() + expect(result.failure?.code).toBe('UNKNOWN_COLUMN') + }) - it('surfaces the FFI ciphertext error code on the bulk decrypt path', async () => { - // The v3 bulk decrypt failure path (resolveDecryptResult + - // throwPreservingCode against a typed client) is otherwise never exercised - // end-to-end. Like the single-item v3 path, a malformed __source is rejected - // as "Invalid EQL ciphertext" with a real FFI code — not the v2 fallback. - const result = await typedDynamo.bulkDecryptModels( - [{ email__source: 'not-a-ciphertext' }], - users, - ) + it('surfaces the FFI ciphertext error code on the bulk decrypt path', async () => { + // The v3 bulk decrypt failure path (resolveDecryptResult + + // throwPreservingCode against a typed client) is otherwise never exercised + // end-to-end. Like the single-item v3 path, a malformed __source is rejected + // as "Invalid EQL ciphertext" with a real FFI code — not the v2 fallback. + const result = await typedDynamo.bulkDecryptModels( + [{ email__source: 'not-a-ciphertext' }], + users, + ) + + expect(result.failure).toBeDefined() + expect(result.failure?.name).toBe('EncryptedDynamoDBError') + expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') + expect(result.failure?.details).toEqual({ context: 'bulkDecryptModels' }) + }) - expect(result.failure).toBeDefined() - expect(result.failure?.name).toBe('EncryptedDynamoDBError') - expect(result.failure?.code).toBe('INVALID_CIPHERTEXT') - expect(result.failure?.details).toEqual({ context: 'bulkDecryptModels' }) + it('routes v3 failures to the configured errorHandler', async () => { + const seen: string[] = [] + const instrumented = encryptedDynamoDB({ + encryptionClient: nominalClient, + options: { errorHandler: (e) => seen.push(e.code) }, }) - it('routes v3 failures to the configured errorHandler', async () => { - const seen: string[] = [] - const instrumented = encryptedDynamoDB({ - encryptionClient: nominalClient, - options: { errorHandler: (e) => seen.push(e.code) }, - }) - - await instrumented.encryptModel({ nope: 'value' }, unknown) + await instrumented.encryptModel({ nope: 'value' }, unknown) - expect(seen).toEqual(['UNKNOWN_COLUMN']) - }) - }, -) + expect(seen).toEqual(['UNKNOWN_COLUMN']) + }) +}) From 23b677a1768e6efcd9a3632c8191aab1d5d3db2b Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 22 Jul 2026 12:45:13 +1000 Subject: [PATCH 20/20] refactor(stack): align DynamoDB isStoredEqlPayload with core predicate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tighten the adapter's local envelope detector to mirror the core isEncryptedPayload shape checks exactly (v must be a number, i must be an object), so the two predicates cannot drift apart. Safe today via the matchColumn gate, but this closes the residual window where a plaintext { v, i } lookalike at a declared column could be split. Raised as non-blocking hardening in review. Also reword the two construct-guard test biome-ignore rationales: the noExplicitAny rule does fire on `as any` in __tests__ (only the GritQL type-erasing-assertions plugin is scoped away from tests), so the directives are required — the old ".test.ts is not typechecked" rationale was inaccurate. --- .../stack/__tests__/dynamodb/construct-guard.test.ts | 4 ++-- packages/stack/src/dynamodb/helpers.ts | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/stack/__tests__/dynamodb/construct-guard.test.ts b/packages/stack/__tests__/dynamodb/construct-guard.test.ts index 65ba86f2..88c95e99 100644 --- a/packages/stack/__tests__/dynamodb/construct-guard.test.ts +++ b/packages/stack/__tests__/dynamodb/construct-guard.test.ts @@ -42,7 +42,7 @@ function stubClient(knownTables: string[]) { bulkEncryptModels: unreached, decryptModel: unreached, bulkDecryptModels: unreached, - // biome-ignore lint/suspicious/noExplicitAny: test stub, .test.ts is not typechecked + // biome-ignore lint/suspicious/noExplicitAny: deliberately permissive test-stub client } as any } @@ -116,7 +116,7 @@ describe('encryptedDynamoDB client/table version guard', () => { bulkEncryptModels: unreached, decryptModel: unreached, bulkDecryptModels: unreached, - // biome-ignore lint/suspicious/noExplicitAny: test stub + // biome-ignore lint/suspicious/noExplicitAny: deliberately permissive test-stub client } as any, }) expect(() => diff --git a/packages/stack/src/dynamodb/helpers.ts b/packages/stack/src/dynamodb/helpers.ts index 55b99bdb..68966d40 100644 --- a/packages/stack/src/dynamodb/helpers.ts +++ b/packages/stack/src/dynamodb/helpers.ts @@ -229,11 +229,19 @@ type StoredEqlPayload = { * enough — `c` is an ordinary attribute name (country, currency, count), and * treating `{ c: 'AU', d: 1 }` as a payload silently rewrites it to * `__source` and DISCARDS every sibling key. + * + * The shape checks (`v` a number, `i` an object) mirror the core + * `isEncryptedPayload` exactly, so the two envelope detectors cannot drift + * apart. The `matchColumn` gate already means this only runs against values at + * a declared encrypted column, but pinning the same predicate closes the + * residual window where a plaintext `{ v, i }` lookalike could be split. */ function isStoredEqlPayload(value: unknown): value is StoredEqlPayload { if (value === null || typeof value !== 'object') return false - if (!('v' in value) || !('i' in value)) return false - return 'c' in value || 'sv' in value + const obj = value as Record + if (!('v' in obj) || typeof obj.v !== 'number') return false + if (!('i' in obj) || typeof obj.i !== 'object') return false + return 'c' in obj || 'sv' in obj } /**