From da4ab9341764eb2d737ba7eb769e153dd320e044 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 26 Sep 2026 15:12:54 +0700 Subject: [PATCH 1/3] fix(plugin-mongodb): write binary, nested and specially named fields from the grid as the values they are --- CHANGELOG.md | 7 + CLAUDE.md | 2 +- .../BsonDocumentFlattener.swift | 41 +- .../MongoDBBinarySubtypes.swift | 67 +++ .../MongoDBCapabilities.swift | 7 + .../MongoDBCollectionDDL.swift | 13 +- .../MongoDBConnection+ScriptHelpers.swift | 12 +- .../MongoDBPluginDriver.swift | 76 +-- .../MongoDBStatementGenerator.swift | 473 ++++++++++++------ .../MongoDBUpdateDocument.swift | 80 +++ .../MongoDBWriteRefusal.swift | 84 ++++ .../MongoDocumentText.swift | 12 + .../MongoExtendedJsonForm.swift | 206 ++++++++ .../MongoNestedValueDiff.swift | 75 +++ .../MongoScriptResultBuilder.swift | 4 +- TablePro/Resources/Localizable.xcstrings | 33 ++ .../Plugins/MongoDBCollectionDDLTests.swift | 4 +- .../Plugins/MongoDBFieldNameUpdateTests.swift | 209 ++++++++ .../MongoDBNestedValueWriteTests.swift | 344 +++++++++++++ .../MongoDBStatementGeneratorTests.swift | 239 +++++---- .../Plugins/MongoDBWriteBackBinaryTests.swift | 233 +++++++++ .../Plugins/MongoDBWriteBackTypeTests.swift | 70 +-- docs/databases/mongodb.mdx | 11 +- project.yml | 6 + 24 files changed, 1937 insertions(+), 371 deletions(-) create mode 100644 Plugins/MongoDBDriverPlugin/MongoDBBinarySubtypes.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoDBUpdateDocument.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoNestedValueDiff.swift create mode 100644 TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift create mode 100644 TableProTests/Plugins/MongoDBNestedValueWriteTests.swift create mode 100644 TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 21b122ab4a..b154604f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pre-connect script failures sometimes reported without the script's own error message. - Failed MongoDB statements, including writes the server rejected, reported as successful with an empty result. - Save reporting success after leaving out an edit it could not write, such as a new MongoDB document left empty. (#3132) +- Binary fields missing from duplicated or pasted MongoDB rows. (#3132) +- Undo of a MongoDB delete refused when the document held binary data. (#3132) +- Binary field deleted when editing a MongoDB binary cell. (#3132) +- Error saving a new MongoDB row with every cell empty. (#3132) +- **Set DEFAULT** on a MongoDB field storing the text `__DEFAULT__`. (#3132) +- Edits to MongoDB fields named with a dot, a leading `$` or `__proto__` changing another field or nothing. (#3132) +- Nested MongoDB values shown with sorted keys, and saved with ObjectIds, dates and numbers retyped. (#3132) - `tablepro-mcp` crashing when its standard input was non-blocking. - `tablepro-mcp` using a full CPU core, or crashing, when its standard output or error was non-blocking. - Server connections piling up while browsing many databases or schemas, and staying open after a failed connect. (#3103) diff --git a/CLAUDE.md b/CLAUDE.md index 6f059affe2..0500cec4d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -238,7 +238,7 @@ To ship one: add the record type or field in CloudKit Console (or `xcrun cktool Four rules follow. A statement whose *answer* is session-scoped is not replayable even though it changes nothing and leaves the footprint clean: `LAST_INSERT_ID`, `ROW_COUNT`, `FOUND_ROWS` and `CONNECTION_ID` are in `mysqlSideEffectingMarkers` for that reason, measured as a fresh connection answering `SELECT LAST_INSERT_ID()` with `0`. The open transaction is the server's own answer rather than a reading of the text, through `mariadb_get_info(MARIADB_CONNECTION_SERVER_STATUS)` in `MariaDBPluginConnection.recordTransactionState` and `footprint.observeServerTransaction`: measured, that reports the transaction `SET autocommit = 0` plus a plain `SELECT` opens, the one inside `/*!40101 BEGIN */` and the one an `XA START` opens, none of which any text scan can see. Only the driver's own statements go in with `countsAsActivity: false`, and only because the driver puts them back itself: the query timeout, and the `USE` behind a database switch, which reconnects through `_activeDatabase` and refreshes `lastActivity` by hand so the idle timer still sees the switch as use. And `/*!40101 ... */` is not a comment, it is SQL the server runs, so `SQLStatementSplitting` keeps it whole rather than stripping it and `MySQLSessionFootprint` reads the body, trailing text included; a mysqldump preamble run from the editor sets the character set, the time zone and eight `@OLD_` variables inside them, and dropping them left the footprint reading clean. The version number is not checked against the server, so a statement the server is too old to run still counts, which holds a clean connection rather than releasing a dirty one. -**A MongoDB update or delete is anchored on `_id` or it does not run**: `generateDelete` used to fall back to a filter built from the remaining columns, which silently dropped every value it could not stringify (all binary) and then `deleteOne`d the first partial match, so a document with a binary `_id` could delete a different document. Both paths now skip with a logged warning instead, matching what `generateUpdate` already did. +**A MongoDB update or delete is anchored on `_id` or it does not run**: `generateDelete` used to fall back to a filter built from the remaining columns, which silently dropped every value it could not stringify (all binary) and then `deleteOne`d the first partial match, so a document with a binary `_id` could delete a different document. A row with no `_id`, or a binary `_id` whose subtype is not known, now throws `PluginRowWriteRefusal` from `generateRowWrites`, so the whole save is refused with the row named rather than skipped. The same goes for every value the shell cannot carry as the grid shows it (`MongoDBWriteRefusal`): a write the generator cannot express faithfully is refused, never left out or written as something else. **Redis Cluster routing follows the server's own answer, and the curated table is only a fallback**: `RedisCommandRouting` fetches `COMMAND` once at connect, which supplies key positions on every Redis and, from Redis 7, the `request_policy` / `response_policy` tips that say which commands fan out and how their replies combine. A policy lives on the *subcommand* entry, not the container (`COMMAND INFO config` carries no tips at all; `config|set` is what says `all_nodes`), so the table is keyed `container|sub`. Redis 6 reports no tips, so a parsed reply is merged *over* the curated table rather than replacing it, or `DBSIZE`, `KEYS` and `FLUSHDB` would each go to one shard of a cluster and report success. The curated table is a hand-written list that has to agree with Redis and that nothing at runtime checks, so `scripts/check-redis-command-routing.sh [host] [port]` diffs it against a live Redis 7+; it found 32 disagreements the first time it ran, including a container command hashed on its literal subcommand name and `MSETNX` marked splittable when splitting it breaks the guarantee it exists for. Two rules follow. A container command takes no key of its own, so `OBJECT`, `MEMORY` and `CONFIG` declare no key positions and their keyed subcommands are listed separately, at the index the key sits in the *full* argument list (`OBJECT ENCODING k` puts it at 2, not 1). And an unknown command is routed as keyless rather than by hashing `argv[1]`: a keyless container like `SCRIPT LOAD` answers `+OK` from one node and never sends a `MOVED` to correct the guess. diff --git a/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift b/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift index 66462cb345..7d655ba854 100644 --- a/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift +++ b/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift @@ -35,6 +35,14 @@ enum BsonValueKind: Hashable { } } +/// Documents as Swift values beside the canonical Extended JSON each was read from. The +/// dictionaries have lost the stored field order and which numeric type each number was; the text +/// has not. +struct MongoReadDocuments { + let dictionaries: [[String: Any]] + let texts: [String] +} + struct BsonDocumentFlattener { // MARK: - Public API @@ -67,22 +75,47 @@ struct BsonDocumentFlattener { } /// Flatten documents into a grid. Missing fields become nil cells. - /// Nested objects/arrays are serialized as compact JSON strings. + /// Nested objects/arrays are serialized as compact JSON strings, taken from each document's + /// stored text when there is one, so they keep their field order and their BSON types. static func flatten( documents: [[String: Any]], columns: [String], kinds: [BsonValueKind], - representation: MongoDBUuidRepresentation + representation: MongoDBUuidRepresentation, + storedTexts: [String] = [] ) -> [[PluginCellValue]] { - documents.map { doc in - columns.enumerated().map { index, column in + documents.enumerated().map { offset, doc in + var stored: [String: MongoDocumentText.Value]? + return columns.enumerated().map { index, column in guard let value = doc[column] else { return PluginCellValue.null } + if isNestedValue(value), offset < storedTexts.count { + if stored == nil { stored = storedMembers(of: storedTexts[offset]) } + if let member = stored?[column] { return .text(nestedDisplayText(member)) } + } let kind = index < kinds.count ? kinds[index] : .string return cellValue(for: value, kind: kind, representation: representation) } } } + /// A document or an array, and not the `$code` or DBRef shapes that render as their own text. + private static func isNestedValue(_ value: Any) -> Bool { + if value is [Any] { return true } + guard let dict = value as? [String: Any] else { return false } + let isCode = dict["$code"] is String + let isReference = dict["$ref"] is String && dict["$id"] != nil + return !isCode && !isReference + } + + private static func storedMembers(of text: String) -> [String: MongoDocumentText.Value] { + guard case .object(let members) = try? MongoDocumentText.Value(parsing: text) else { return [:] } + return Dictionary(members.map { ($0.key, $0.value) }, uniquingKeysWith: { first, _ in first }) + } + + static func nestedDisplayText(_ canonical: MongoDocumentText.Value) -> String { + JSONTruncation.truncate(MongoExtendedJsonForm.display(canonical).compactText, maxLength: maxNestedJsonLength) + } + /// Infer the dominant value kind for each column by majority-vote over document values. static func columnKinds( for columns: [String], diff --git a/Plugins/MongoDBDriverPlugin/MongoDBBinarySubtypes.swift b/Plugins/MongoDBDriverPlugin/MongoDBBinarySubtypes.swift new file mode 100644 index 0000000000..1cca946afa --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBBinarySubtypes.swift @@ -0,0 +1,67 @@ +// +// MongoDBBinarySubtypes.swift +// MongoDBDriverPlugin +// + +import CryptoKit +import Foundation +import TableProPluginKit + +/// The BSON subtype of each binary value the grid was handed, found again by the value itself. +/// +/// A grid cell carries binary data as bytes alone, and a write has to send the subtype back. A +/// column-wide answer is wrong twice over: one field can hold several subtypes, and the kinds a +/// driver remembers are overwritten by whichever result it built last, which may be another tab's. +/// Keying by the field and a digest of the bytes ties the subtype to the value the user copied or +/// edited, and a value seen with two subtypes answers nothing rather than a guess. +struct MongoDBBinarySubtypes: Sendable { + private var subtypesByField: [String: [Data: Set]] = [:] + private(set) var count = 0 + + static let empty = MongoDBBinarySubtypes() + + var isEmpty: Bool { subtypesByField.isEmpty } + + /// Every top-level binary value in the documents, which is every value a cell can hold as bytes. + static func recording(_ documents: [[String: Any]]) -> MongoDBBinarySubtypes { + var recorded = MongoDBBinarySubtypes() + for document in documents { + for (field, value) in document { + if let binary = value as? MongoDBBinaryValue { + recorded.record(binary.data, subtype: binary.subtype, field: field) + } else if let data = value as? Data { + recorded.record(data, subtype: 0, field: field) + } + } + } + return recorded + } + + mutating func record(_ data: Data, subtype: UInt8, field: String) { + insert(subtype, digest: Self.digest(of: data), field: field) + } + + /// Every subtype the value was seen with in this field: one is the answer, none or several are not. + func subtypes(of data: Data, in field: String) -> Set { + subtypesByField[field]?[Self.digest(of: data)] ?? [] + } + + func merging(_ other: MongoDBBinarySubtypes) -> MongoDBBinarySubtypes { + var merged = self + for (field, digests) in other.subtypesByField { + for (digest, subtypes) in digests { + subtypes.forEach { merged.insert($0, digest: digest, field: field) } + } + } + return merged + } + + private mutating func insert(_ subtype: UInt8, digest: Data, field: String) { + guard subtypesByField[field, default: [:]][digest, default: []].insert(subtype).inserted else { return } + count += 1 + } + + private static func digest(of data: Data) -> Data { + Data(SHA256.hash(data: data)) + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDBCapabilities.swift b/Plugins/MongoDBDriverPlugin/MongoDBCapabilities.swift index 0805bc8d16..dfd8b5cf71 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBCapabilities.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBCapabilities.swift @@ -19,6 +19,13 @@ struct MongoDBCapabilities: Sendable, Equatable { major >= 4 } + /// `$setField` and `$unsetField` arrived in 5.0. Nil when the version is not known, so the + /// server answers for itself. + var supportsFieldExpressions: Bool? { + guard self != .unknown else { return nil } + return major >= 5 + } + static func parse(_ version: String?) -> MongoDBCapabilities { guard let version else { return .unknown } let parts = version.split(separator: ".") diff --git a/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift b/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift index 21d80c4627..556e781658 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift @@ -114,15 +114,18 @@ enum MongoDBCollectionDDL { return nil } - /// The statement is a JavaScript object literal, and JavaScript lists integer-like keys first in + /// The statement is a JavaScript object literal, and JavaScript lists array-index keys first in /// ascending order and reads `__proto__` as the prototype rather than as a key, so neither kind of - /// name reaches the server where it was written. - private static func isReorderedByTheShell(_ name: String) -> Bool { + /// name reaches the server where it was written. An array index is an integer from 0 to + /// 4294967294 written without a leading zero; a larger one stays where it was written. + static func isReorderedByTheShell(_ name: String) -> Bool { if name == "__proto__" { return true } - guard !name.isEmpty, name.utf8.allSatisfy({ (UInt8(ascii: "0") ... UInt8(ascii: "9")).contains($0) }) else { + guard name.utf8.allSatisfy({ (UInt8(ascii: "0") ... UInt8(ascii: "9")).contains($0) }), + name == "0" || !name.hasPrefix("0"), + let index = UInt32(name) else { return false } - return name == "0" || !name.hasPrefix("0") + return index < UInt32.max } // MARK: - Indexes diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift index a8ab73b585..907d68cdde 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection+ScriptHelpers.swift @@ -16,13 +16,17 @@ struct MongoScriptDocumentBatch: Sendable { var jsonArray: String { "[\(json.joined(separator: ","))]" } - var dictionaries: [[String: Any]] { - json.compactMap { document in + var readDocuments: MongoReadDocuments { + var dictionaries: [[String: Any]] = [] + var texts: [String] = [] + for document in json { guard let data = document.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: data), - let dictionary = object as? [String: Any] else { return nil } - return MongoDBConnection.unwrapExtendedJson(dictionary) as? [String: Any] ?? dictionary + let dictionary = object as? [String: Any] else { continue } + dictionaries.append(MongoDBConnection.unwrapExtendedJson(dictionary) as? [String: Any] ?? dictionary) + texts.append(document) } + return MongoReadDocuments(dictionaries: dictionaries, texts: texts) } } diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 5c6da17ebe..c1f1aef7c9 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -20,6 +20,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var fieldPathKindsByCollection: [String: [String: BsonValueKind]] = [:] private var declaredSchemasByCollection: [String: MongoDBCollectionSchema] = [:] private var identityKindsByCollection: [String: BsonValueKind] = [:] + private var binarySubtypesByCollection: [String: MongoDBBinarySubtypes] = [:] private static let logger = Logger(subsystem: "com.TablePro", category: "MongoDBPluginDriver") @@ -784,20 +785,17 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } - func generateStatements( + func generateRowWrites( table: String, + schema: String?, columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set - ) -> [(statement: String, parameters: [PluginCellValue])]? { - let generator = MongoDBStatementGenerator( - collectionName: table, columns: columns, columnKinds: columnKinds(for: table), - declaredKinds: declaredKinds(for: table), identityKind: identityKind(for: table) - ) - return generator.generateStatements( + ) throws -> [PluginRowWrite]? { + try writeGenerator(for: table, columns: columns).generateRowWrites( from: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, insertedRowIndices: insertedRowIndices ) @@ -810,11 +808,28 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { primaryKeyColumns: [String], rows: [[PluginCellValue]] ) -> [(statement: String, parameters: [PluginCellValue])]? { - let generator = MongoDBStatementGenerator( - collectionName: table, columns: columns, columnKinds: columnKinds(for: table), - declaredKinds: declaredKinds(for: table), identityKind: identityKind(for: table) - ) - return generator.generateRestore(rows: rows) + writeGenerator(for: table, columns: columns).generateRestore(rows: rows) + } + + private func writeGenerator(for table: String, columns: [String]) -> MongoDBStatementGenerator { + let key = columnKindKey(table) + return columnKindLock.withLock { + let declared = declaredSchemasByCollection[key] ?? .empty + return MongoDBStatementGenerator( + collectionName: table, + columns: columns, + columnKinds: columnKindsByCollection[key] ?? [:], + declaredKinds: declared.valueKinds, + identityKind: identityKindsByCollection[key], + binarySubtypes: binarySubtypesByCollection[key] ?? .empty, + declaredBinaryFields: Self.binaryFields(declaredBy: declared), + capabilities: { MongoDBCapabilities.parse(self.serverVersion) } + ) + } + } + + private static func binaryFields(declaredBy schema: MongoDBCollectionSchema) -> Set { + Set(schema.fields.filter { $0.bsonTypes.filter { $0 != "null" } == ["binData"] }.map(\.name)) } // MARK: - Streaming @@ -874,7 +889,8 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } /// Hands over a statement that had already run by the time the export asked, rather than - /// running it a second time. + /// running it a second time. Nested values render the way a streamed cursor renders them, so + /// one export never mixes two spellings. private func yieldMaterialised( _ outcome: MongoScriptStatementResult, into continuation: AsyncThrowingStream.Continuation @@ -884,7 +900,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { startTime: Date(), documents: { documents, collection, isTruncated in self.buildPluginResult( - from: documents, startTime: Date(), + from: MongoReadDocuments(dictionaries: documents.dictionaries, texts: []), startTime: Date(), isTruncated: isTruncated, collection: collection ) } @@ -903,12 +919,13 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Result Building private func buildPluginResult( - from documents: [[String: Any]], + from read: MongoReadDocuments, startTime: Date, isTruncated: Bool = false, collection: String = "", declared: MongoDBCollectionSchema = .empty ) -> PluginQueryResult { + let documents = read.dictionaries if documents.isEmpty { return PluginQueryResult( columns: [], columnTypeNames: [], @@ -924,13 +941,15 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { rememberColumnKinds(sampledKinds, for: sampled, collection: collection) rememberIdentityKind(of: documents, collection: collection) rememberFieldPathKinds(from: documents, collection: collection) + rememberBinarySubtypes(of: documents, collection: collection) let unseen = MongoDBCollectionShape.declaredColumnsMissing(from: sampled, schema: declared) let columns = sampled + unseen let kinds = sampledKinds + unseen.map { declared.field(named: $0)?.valueKind ?? .null } let typeNames = sampledKinds.map { BsonDocumentFlattener.typeName(for: $0, representation: uuidRepresentation) } + unseen.map { declaredTypeName(of: $0, in: declared) } let rows = BsonDocumentFlattener.flatten( - documents: documents, columns: columns, kinds: kinds, representation: uuidRepresentation + documents: documents, columns: columns, kinds: kinds, + representation: uuidRepresentation, storedTexts: read.texts ) return PluginQueryResult( @@ -984,11 +1003,22 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { columnKindLock.withLock { columnKindsByCollection[key] = byName } } - private func columnKinds(for collection: String) -> [String: BsonValueKind] { + /// Added to rather than replaced, so a value keeps its subtype after another page or another tab + /// of the same collection is read, and a value seen with two subtypes stays ambiguous. Past the + /// cap the collection starts over from this page: a value no longer found is refused, not guessed. + private func rememberBinarySubtypes(of documents: [[String: Any]], collection: String) { + guard !collection.isEmpty else { return } + let page = MongoDBBinarySubtypes.recording(documents) + guard !page.isEmpty else { return } let key = columnKindKey(collection) - return columnKindLock.withLock { columnKindsByCollection[key] ?? [:] } + columnKindLock.withLock { + let merged = (binarySubtypesByCollection[key] ?? .empty).merging(page) + binarySubtypesByCollection[key] = merged.count > Self.binarySubtypeLimit ? page : merged + } } + private static let binarySubtypeLimit = 50_000 + private func rememberIdentityKind(of documents: [[String: Any]], collection: String) { guard !collection.isEmpty else { return } let kind = BsonDocumentFlattener.uniformKind( @@ -998,16 +1028,6 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { columnKindLock.withLock { identityKindsByCollection[key] = kind } } - private func identityKind(for collection: String) -> BsonValueKind? { - let key = columnKindKey(collection) - return columnKindLock.withLock { identityKindsByCollection[key] } - } - - private func declaredKinds(for collection: String) -> [String: BsonValueKind] { - let key = columnKindKey(collection) - return columnKindLock.withLock { declaredSchemasByCollection[key]?.valueKinds ?? [:] } - } - /// Recorded from the documents a browse already fetched, on the session driver that will /// build the filter. Sampling through `sampleFieldPaths` cannot do it: that call is routed /// through `MetadataConnectionPool`, so it lands on a different driver instance whose cache diff --git a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index 90bb05085d..64b611aae3 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -2,7 +2,7 @@ // MongoDBStatementGenerator.swift // MongoDBDriverPlugin // -// Generates MongoDB shell commands (insertOne, replaceOne, deleteOne) from tracked changes. +// Generates MongoDB shell commands (insertOne, updateOne, deleteOne, deleteMany) from tracked changes. // Plugin-local version using PluginRowChange instead of Core types. // @@ -25,219 +25,373 @@ struct MongoDBStatementGenerator { /// match nothing. var identityKind: BsonValueKind? + /// The subtype of each binary value the grid shows, so bytes written back keep the subtype they + /// were read with. + var binarySubtypes: MongoDBBinarySubtypes = .empty + /// Fields the validator declares as binary. New bytes with no earlier value there are generic + /// binary, subtype 0, which is also what the column's `BLOB` type name says. + var declaredBinaryFields: Set = [] + /// Asked only for a row that needs `$setField`, since reading the version can wait on the + /// connection. + var capabilities: () -> MongoDBCapabilities = { .unknown } + + private static let defaultMarker = "__DEFAULT__" + private var collectionAccessor: String { MongoCollectionAccessor.expression(for: collectionName) } /// Index of "_id" field in the columns array (used as primary key equivalent) var idColumnIndex: Int? { - columns.firstIndex(of: "_id") + columns.firstIndex(of: MongoDBCollectionDDL.idField) } // MARK: - Public API - /// Generate MongoDB shell statements from changes - func generateStatements( + /// The statements for a save, each naming the change it writes. + /// + /// A change the shell cannot carry as the value the grid shows is refused with the reason, + /// never left out and never written as something else. + func generateRowWrites( from changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set - ) -> [(statement: String, parameters: [PluginCellValue])] { - var statements: [(statement: String, parameters: [PluginCellValue])] = [] - var deleteChanges: [PluginRowChange] = [] + ) throws -> [PluginRowWrite] { + var writes: [PluginRowWrite] = [] + var deletions: [(rowIndex: Int, identity: String)] = [] for change in changes { - switch change.type { - case .insert: - guard insertedRowIndices.contains(change.rowIndex) else { continue } - if let stmt = generateInsert(for: change, insertedRowData: insertedRowData) { - statements.append(stmt) + do { + switch change.type { + case .insert: + guard insertedRowIndices.contains(change.rowIndex) else { continue } + let statement = try insertStatement(for: change, insertedRowData: insertedRowData) + writes.append(PluginRowWrite(statement: statement, rowIndices: [change.rowIndex])) + case .update: + guard let statement = try updateStatement(for: change) else { continue } + writes.append(PluginRowWrite(statement: statement, rowIndices: [change.rowIndex])) + case .delete: + guard deletedRowIndices.contains(change.rowIndex) else { continue } + deletions.append((rowIndex: change.rowIndex, identity: try identityJson(of: change))) } - case .update: - if let stmt = generateUpdate(for: change) { - statements.append(stmt) - } - case .delete: - guard deletedRowIndices.contains(change.rowIndex) else { continue } - deleteChanges.append(change) + } catch let refusal as MongoDBWriteRefusal { + throw refusal.refusal(ofRow: change.rowIndex) } } - // Batch deletes into a single deleteMany when possible - if let bulkDelete = generateBulkDelete(from: deleteChanges) { - statements.append(bulkDelete) - } else { - for change in deleteChanges { - if let stmt = generateDelete(for: change) { - statements.append(stmt) - } - } + if let deletion = deleteWrite(for: deletions) { + writes.append(deletion) } - - return statements + return writes } // MARK: - INSERT - private func generateInsert( + /// NULL and DEFAULT leave a field out of a new document. A row with nothing else in it is the + /// empty document, which the server stores with a generated `_id`. + private func insertStatement( for change: PluginRowChange, insertedRowData: [Int: [PluginCellValue]] - ) -> (statement: String, parameters: [PluginCellValue])? { - var doc: [String: String] = [:] - + ) throws -> String { + let cells: [(field: String, value: PluginCellValue)] if let values = insertedRowData[change.rowIndex] { - for (index, value) in values.enumerated() { - guard index < columns.count else { continue } - let column = columns[index] - // Skip _id for inserts (let MongoDB auto-generate) - if column == "_id" { continue } - // Skip DEFAULT sentinel - let textValue = value.asText - if textValue == "__DEFAULT__" { continue } - if let val = textValue { - doc[column] = val - } - } + cells = zip(columns, values).map { (field: $0, value: $1) } } else { - // Fallback: use cellChanges - for cellChange in change.cellChanges { - if cellChange.columnName == "_id" { continue } - let newText = cellChange.newValue.asText - if newText == "__DEFAULT__" { continue } - if let val = newText { - doc[cellChange.columnName] = val - } - } + cells = change.cellChanges.map { (field: $0.columnName, value: $0.newValue) } } + let entries = try cells.filter { !isLeftOut($0.value) }.map { cell in + "\(quotedKey(cell.field)): \(try documentValueJson(cell.value, field: cell.field))" + } + return "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" + } - guard !doc.isEmpty else { return nil } + private func isLeftOut(_ value: PluginCellValue) -> Bool { + value.isNull || value.asText == Self.defaultMarker + } - guard let docJson = serializeDocument(doc) else { return nil } - let shell = "\(collectionAccessor).insertOne(\(docJson))" - return (statement: shell, parameters: []) + /// A value of a whole document the grid writes: a new row, a duplicate or a paste, or a + /// deleted document put back. An `_id` the row holds is kept, typed the way the filters type it; + /// the grid leaves a new row's `_id` as DEFAULT, so the server generates it. + private func documentValueJson(_ value: PluginCellValue, field: String) throws -> String { + if field == MongoDBCollectionDDL.idField { + return try idValueJson(value) + } + guard !field.isEmpty, field != "__proto__" else { + throw MongoDBWriteRefusal.unwritableFieldName(field: field) + } + return try valueJson(value, field: field, replacing: nil) } // MARK: - Restore /// Puts a deleted document back with the `_id` it had. /// - /// `generateInsert` drops `_id` so a row the user just added gets a server-generated one. - /// Undoing a delete is the opposite requirement: a new `_id` is a different document, and - /// anything that referenced the old one still points at nothing. + /// A new row leaves `_id` to the server. Undoing a delete is the opposite requirement: a new + /// `_id` is a different document, and anything that referenced the old one still points at + /// nothing. A value that cannot be written refuses the restore rather than dropping the field. func generateRestore(rows: [[PluginCellValue]]) -> [(statement: String, parameters: [PluginCellValue])]? { guard let idIndex = idColumnIndex else { return nil } - var statements: [(statement: String, parameters: [PluginCellValue])] = [] - for row in rows { - guard idIndex < row.count, let idValue = row[idIndex].asText else { return nil } - - var doc: [String: String] = [:] - for (index, value) in row.enumerated() where index != idIndex { - guard index < columns.count else { continue } - /// A binary field has no text form here, and writing the document without it - /// restores a document that is missing a field. Refuse the whole restore instead, - /// which the host reports rather than passing off as a success. - if value.asBytes != nil { return nil } - guard let text = value.asText else { continue } - if text == "__DEFAULT__" { continue } - doc[columns[index]] = text + do { + return try rows.map { row in + guard idIndex < row.count else { throw MongoDBWriteRefusal.missingIdentity } + var entries = ["\"_id\": \(try idValueJson(row[idIndex]))"] + for (index, value) in row.enumerated() where index != idIndex && index < columns.count { + guard !isLeftOut(value) else { continue } + let field = columns[index] + entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field))") + } + let statement = "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" + return (statement: statement, parameters: []) } - - guard var docJson = serializeDocument(doc) else { return nil } - let idEntry = "\"_id\": \(idValueJson(idValue))" - docJson = docJson == "{}" ? "{\(idEntry)}" : "{\(idEntry), " + String(docJson.dropFirst()) - statements.append((statement: "\(collectionAccessor).insertOne(\(docJson))", parameters: [])) + } catch let refusal as MongoDBWriteRefusal { + Self.logger.warning( + "Refusing to restore into '\(self.collectionName, privacy: .private)': \(refusal.reason, privacy: .public)" + ) + return nil + } catch { + return nil } - return statements } - // MARK: - UPDATE (updateOne with $set/$unset) + // MARK: - UPDATE - private func generateUpdate(for change: PluginRowChange) -> (statement: String, parameters: [PluginCellValue])? { - guard !change.cellChanges.isEmpty else { return nil } + private enum CellWrite { + case remove(field: String) + case whole(field: String, json: String) + case nested(field: String, changes: [MongoNestedValueDiff.Change], edited: MongoDocumentText.Value) - guard let idIndex = idColumnIndex, - let originalRow = change.originalRow, - idIndex < originalRow.count, - let idValue = originalRow[idIndex].asText else { - Self.logger.warning("Skipping UPDATE for collection '\(self.collectionName)' - no _id value") - return nil + var field: String { + switch self { + case .remove(let field), .whole(let field, _), .nested(let field, _, _): return field + } } + } - var setDoc: [String: String] = [:] - var unsetFields: [String] = [] + private func updateStatement(for change: PluginRowChange) throws -> String? { + guard !change.cellChanges.isEmpty else { return nil } + let identity = try identityJson(of: change) + let cellWrites = try change.cellChanges.map { + try cellWrite(field: $0.columnName, from: $0.oldValue, to: $0.newValue) + } + let update = try updateDocument(for: cellWrites) + return "\(collectionAccessor).updateOne({\"_id\": \(identity)}, \(update))" + } - for cellChange in change.cellChanges { - if cellChange.columnName == "_id" { continue } - if let val = cellChange.newValue.asText { - setDoc[cellChange.columnName] = val - } else { - unsetFields.append(cellChange.columnName) + private func cellWrite(field: String, from oldValue: PluginCellValue, to newValue: PluginCellValue) throws -> CellWrite { + guard field != MongoDBCollectionDDL.idField else { throw MongoDBWriteRefusal.identityChanged } + switch newValue { + case .null: + return .remove(field: field) + case .bytes(let data): + return .whole(field: field, json: try binaryJson(data, field: field, replacing: oldValue)) + case .text(let text): + guard text != Self.defaultMarker else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } + if let nested = try nestedEdit(of: field, from: oldValue, to: text) { + return nested } + return .whole(field: field, json: try valueJson(newValue, field: field, replacing: oldValue)) } + } - guard !setDoc.isEmpty || !unsetFields.isEmpty else { return nil } + /// An edit of a nested document or array, as the paths it changed. + /// + /// Nil when the cell does not hold a complete one on both sides, and the edit is then written + /// whole. An old value shortened for display cannot be diffed, but a complete value typed over + /// it replaces the field without needing it; new text that is still shortened is refused when + /// it is written. + private func nestedEdit(of field: String, from oldValue: PluginCellValue, to text: String) throws -> CellWrite? { + guard isContainerKind(field), case .text(let oldText) = oldValue, opensContainer(text), + !JSONTruncation.isIncompleteStructure(text), !JSONTruncation.isIncompleteStructure(oldText), + let old = try? MongoDocumentText.Value(parsing: oldText), old.isContainer else { + return nil + } + guard let edited = try? MongoDocumentText.Value(parsing: text), edited.isContainer else { + throw MongoDBWriteRefusal.unreadableJSON(field: field) + } + let changes = MongoNestedValueDiff.changes(from: old, to: edited, at: field) + return .nested(field: field, changes: changes, edited: edited) + } - let filterJson = buildIdFilter(idValue) + /// A classic update when every field can be named by a path, otherwise a pipeline that writes + /// each changed field whole. + private func updateDocument(for cellWrites: [CellWrite]) throws -> String { + if let special = cellWrites.first(where: { MongoDBUpdateDocument.needsFieldExpression($0.field) }) { + guard capabilities().supportsFieldExpressions != false else { + throw MongoDBWriteRefusal.fieldNeedsMongoDB5(field: special.field) + } + return MongoDBUpdateDocument.pipeline(try cellWrites.map(wholeFieldWrite)) + } - // Build update document with $set and/or $unset - var updateParts: [String] = [] - if !setDoc.isEmpty { - guard let setJson = serializeDocument(setDoc) else { return nil } - updateParts.append("\"$set\": \(setJson)") + var sets: [(path: String, json: String)] = [] + var removals: [String] = [] + for cellWrite in cellWrites { + switch cellWrite { + case .remove(let field): + removals.append(field) + case .whole(let field, let json): + sets.append((path: field, json: json)) + case .nested(let field, let changes, let edited): + guard !changes.isEmpty else { + sets.append((path: field, json: try shellJson(edited, field: field))) + continue + } + for change in changes { + switch change { + case .set(let path, let value): + sets.append((path: path, json: try shellJson(value, field: field))) + case .remove(let path): + removals.append(path) + } + } + } } - if !unsetFields.isEmpty { - let unsetDoc = unsetFields.sorted().map { "\"\(escapeJsonString($0))\": \"\"" }.joined(separator: ", ") - updateParts.append("\"$unset\": {\(unsetDoc)}") + return MongoDBUpdateDocument.classic(sets: sets, removals: removals) + } + + private func wholeFieldWrite(_ cellWrite: CellWrite) throws -> MongoDBUpdateDocument.FieldWrite { + switch cellWrite { + case .remove(let field): + return .remove(field: field) + case .whole(let field, let json): + return .set(field: field, json: json) + case .nested(let field, _, let edited): + return .set(field: field, json: try shellJson(edited, field: field)) } + } + + // MARK: - DELETE - let updateJson = "{\(updateParts.joined(separator: ", "))}" - let shell = "\(collectionAccessor).updateOne(\(filterJson), \(updateJson))" - return (statement: shell, parameters: []) + /// One `deleteMany` over every `_id` when several rows go, which names each of them. + private func deleteWrite(for deletions: [(rowIndex: Int, identity: String)]) -> PluginRowWrite? { + guard let only = deletions.first else { return nil } + let rowIndices = deletions.map(\.rowIndex) + guard deletions.count > 1 else { + return PluginRowWrite( + statement: "\(collectionAccessor).deleteOne({\"_id\": \(only.identity)})", + rowIndices: rowIndices + ) + } + let inList = deletions.map(\.identity).joined(separator: ", ") + return PluginRowWrite( + statement: "\(collectionAccessor).deleteMany({\"_id\": {\"$in\": [\(inList)]}})", + rowIndices: rowIndices + ) } - // MARK: - DELETE MANY + // MARK: - Values - /// Batch multiple deletes into a single deleteMany with $in when all rows have _id - private func generateBulkDelete(from changes: [PluginRowChange]) -> (statement: String, parameters: [PluginCellValue])? { - guard changes.count > 1, let idIndex = idColumnIndex else { return nil } + private func identityJson(of change: PluginRowChange) throws -> String { + guard let idIndex = idColumnIndex, let originalRow = change.originalRow, idIndex < originalRow.count else { + throw MongoDBWriteRefusal.missingIdentity + } + return try idValueJson(originalRow[idIndex]) + } - var idValues: [String] = [] - for change in changes { - guard let originalRow = change.originalRow, - idIndex < originalRow.count, - let idValue = originalRow[idIndex].asText else { - return nil + /// A cell's value as the JSON the statement carries. + private func valueJson(_ value: PluginCellValue, field: String, replacing oldValue: PluginCellValue?) throws -> String { + switch value { + case .null: + return "null" + case .bytes(let data): + return try binaryJson(data, field: field, replacing: oldValue) + case .text(let text): + if case .bytes(let oldData) = oldValue { + return try textIntoBinaryJson(text, field: field, replacing: oldData) + } + guard !JSONTruncation.isIncompleteStructure(text) else { + throw MongoDBWriteRefusal.truncatedValue(field: field) } - idValues.append(idValueJson(idValue)) + return try jsonValue(for: text, field: field) } + } - let inList = idValues.joined(separator: ", ") - let shell = "\(collectionAccessor).deleteMany({\"_id\": {\"$in\": [\(inList)]}})" - return (statement: shell, parameters: []) + private func binaryJson(_ data: Data, field: String, replacing oldValue: PluginCellValue?) throws -> String { + let subtype = try binarySubtype(of: data, field: field, replacing: oldValue) + return MongoDBUuidCodec.extendedJson(for: MongoDBBinaryValue(data: data, subtype: subtype)) } - // MARK: - DELETE + /// Edited bytes keep the subtype of the value they replace. Copied bytes keep the subtype they + /// were read with. Bytes with neither are generic binary only where the validator declares the + /// field binary; anywhere else the subtype is unknown and the write is refused. + private func binarySubtype(of data: Data, field: String, replacing oldValue: PluginCellValue?) throws -> UInt8 { + if case .bytes(let oldData) = oldValue { + return try onlySubtype(binarySubtypes.subtypes(of: oldData, in: field), field: field) + } + let known = binarySubtypes.subtypes(of: data, in: field) + if known.isEmpty, declaredBinaryFields.contains(field) { + return 0 + } + return try onlySubtype(known, field: field) + } - private func generateDelete(for change: PluginRowChange) -> (statement: String, parameters: [PluginCellValue])? { - guard let originalRow = change.originalRow, - let idIndex = idColumnIndex, - idIndex < originalRow.count, - let idValue = originalRow[idIndex].asText else { - Self.logger.warning("Skipping DELETE for collection '\(self.collectionName)' - no _id value") - return nil + private func onlySubtype(_ subtypes: Set, field: String) throws -> UInt8 { + guard subtypes.count == 1, let only = subtypes.first else { + throw MongoDBWriteRefusal.binarySubtypeUnknown(field: field) } + return only + } - let filterJson = buildIdFilter(idValue) - let shell = "\(collectionAccessor).deleteOne(\(filterJson))" - return (statement: shell, parameters: []) + /// Text typed over a binary value is a UUID in one of its wrappers, or nothing at all. Any other + /// text would turn the field into a string. + private func textIntoBinaryJson(_ text: String, field: String, replacing oldData: Data) throws -> String { + if let wrapped = MongoDBUuidCodec.extendedJsonFromWrapper(text) { + return wrapped + } + guard text.isEmpty else { throw MongoDBWriteRefusal.binaryNeedsBytes(field: field) } + return try binaryJson(Data(), field: field, replacing: .bytes(oldData)) } - // MARK: - Helpers + private func shellJson(_ value: MongoDocumentText.Value, field: String) throws -> String { + try MongoExtendedJsonForm.shellValue(value, field: field).compactText + } - /// Build a filter document for an _id value (Extended JSON for driver execution). - private func buildIdFilter(_ idValue: String) -> String { - "{\"_id\": \(idValueJson(idValue))}" + private func isContainerKind(_ field: String) -> Bool { + let kind = kind(of: field) + return kind == .document || kind == .array + } + + private func opensContainer(_ text: String) -> Bool { + text.hasPrefix("{") || text.hasPrefix("[") + } + + private func quotedKey(_ field: String) -> String { + "\"\(escapeJsonString(field))\"" + } + + // MARK: - Identity + + /// An `_id` held as bytes filters on binary with the subtype it was read with, or the one every + /// sampled `_id` shares. + private func idValueJson(_ value: PluginCellValue) throws -> String { + let idField = MongoDBCollectionDDL.idField + switch value { + case .null: + throw MongoDBWriteRefusal.missingIdentity + case .bytes(let data): + let known = binarySubtypes.subtypes(of: data, in: idField) + if known.isEmpty, case .binary(let subtype) = identityKind { + return MongoDBUuidCodec.extendedJson(for: MongoDBBinaryValue(data: data, subtype: subtype)) + } + let subtype = try onlySubtype(known, field: idField) + return MongoDBUuidCodec.extendedJson(for: MongoDBBinaryValue(data: data, subtype: subtype)) + case .text(let text): + if let document = try documentIdJson(text) { + return document + } + return idValueJson(text) + } + } + + /// An embedded document compares field by field in order, so a document `_id` has to be sent in + /// the order it is stored, which is the order the grid now shows. + private func documentIdJson(_ text: String) throws -> String? { + guard (declaredKinds[MongoDBCollectionDDL.idField] ?? identityKind) == .document, + let parsed = try? MongoDocumentText.Value(parsing: text), case .object = parsed else { + return nil + } + return try shellJson(parsed, field: MongoDBCollectionDDL.idField) } /// An `_id` is matched by value and type together, so a string `_id` of `1001` written as a @@ -277,27 +431,12 @@ struct MongoDBStatementGenerator { return nsValue.length == 24 && value.allSatisfy { $0.isHexDigit } } - /// Serialize a [String: String] dictionary to JSON-like format - private func serializeDocument(_ doc: [String: String]) -> String? { - var entries: [String] = [] - for (key, value) in doc.sorted(by: { $0.key < $1.key }) { - guard !JSONTruncation.isIncompleteStructure(value) else { - Self.logger.warning( - "Skipping write for '\(self.collectionName).\(key)' - the shown value is truncated" - ) - return nil - } - entries.append("\"\(escapeJsonString(key))\": \(jsonValue(for: value, field: key))") - } - return "{\(entries.joined(separator: ", "))}" - } - /// A cell's text as the value it stands for, in the field's type when that type is known. /// /// The statement is JavaScript the shell evaluates, so text is only ever pasted in when it is /// strict JSON. A stored string that merely starts with `[` and ends with `]` would otherwise run /// as code the moment its row is duplicated or its delete is undone. - private func jsonValue(for value: String, field: String) -> String { + private func jsonValue(for value: String, field: String) throws -> String { if declaredKinds[field] == .string { return "\"\(escapeJsonString(value))\"" } @@ -313,8 +452,8 @@ struct MongoDBStatementGenerator { if let binary = MongoDBUuidCodec.extendedJsonFromWrapper(value) { return binary } - if isStrictJsonContainer(value) { - return value + if let container = try containerJson(value, field: field) { + return container } return "\"\(escapeJsonString(value))\"" } @@ -354,11 +493,17 @@ struct MongoDBStatementGenerator { private static let largestExactDouble: UInt64 = 1 << 53 - private func isStrictJsonContainer(_ value: String) -> Bool { - guard (value.hasPrefix("{") && value.hasSuffix("}")) || (value.hasPrefix("[") && value.hasSuffix("]")), - let data = value.data(using: .utf8), - let parsed = try? JSONSerialization.jsonObject(with: data) else { return false } - return parsed is [String: Any] || parsed is [Any] + /// A nested document or array, spelled so the shell stores the types its text shows. Text that + /// only looks like one is a string, except in a field that holds documents or arrays, where it + /// is refused rather than stored as a string. + private func containerJson(_ value: String, field: String) throws -> String? { + guard opensContainer(value) else { return nil } + guard value.hasSuffix("}") || value.hasSuffix("]"), + let parsed = try? MongoDocumentText.Value(parsing: value), parsed.isContainer else { + if isContainerKind(field) { throw MongoDBWriteRefusal.unreadableJSON(field: field) } + return nil + } + return try shellJson(parsed, field: field) } /// Escape special characters for JSON strings (handles Unicode control chars U+0000-U+001F) diff --git a/Plugins/MongoDBDriverPlugin/MongoDBUpdateDocument.swift b/Plugins/MongoDBDriverPlugin/MongoDBUpdateDocument.swift new file mode 100644 index 0000000000..21ddc044ab --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBUpdateDocument.swift @@ -0,0 +1,80 @@ +// +// MongoDBUpdateDocument.swift +// MongoDBDriverPlugin +// + +import Foundation + +/// The update a grid edit sends to `updateOne`. +/// +/// An ordinary edit is a classic `$set`/`$unset` document, which every server version reads. The +/// server reads each key of that document as a path, though, so a field whose own name holds a dot, +/// starts with `$` or is empty cannot be named there, and `__proto__` never reaches the server at +/// all because the statement is a JavaScript object. A row that touches such a field is sent as an +/// aggregation pipeline instead, which names it with `$setField` or `$unsetField` (MongoDB 5.0). +/// +/// Inside a pipeline a dotted path means something else: `$set` of `tags.1` rewrites every element +/// of `tags` rather than the second one, and still reports success. So a pipeline only ever writes +/// whole top-level fields, and every value in it is wrapped in `$literal`, which keeps a string +/// such as `"$price"` or an object shaped like an operator from being evaluated. +enum MongoDBUpdateDocument { + enum FieldWrite: Equatable { + case set(field: String, json: String) + case remove(field: String) + + var field: String { + switch self { + case .set(let field, _), .remove(let field): return field + } + } + } + + static func needsFieldExpression(_ field: String) -> Bool { + !BsonDocumentFlattener.isAddressableSegment(field) || field == "__proto__" + } + + static func classic(sets: [(path: String, json: String)], removals: [String]) -> String { + var parts: [String] = [] + if !sets.isEmpty { + let entries = sets.sorted { $0.path < $1.path }.map { "\(MongoDocumentText.quoted($0.path)): \($0.json)" } + parts.append("\"$set\": {\(entries.joined(separator: ", "))}") + } + if !removals.isEmpty { + let entries = removals.sorted().map { "\(MongoDocumentText.quoted($0)): \"\"" } + parts.append("\"$unset\": {\(entries.joined(separator: ", "))}") + } + return "{\(parts.joined(separator: ", "))}" + } + + static func pipeline(_ writes: [FieldWrite]) -> String { + var stages: [String] = [] + let plainSets = writes.compactMap { write -> String? in + guard case .set(let field, let json) = write, !needsFieldExpression(field) else { return nil } + return "\(MongoDocumentText.quoted(field)): {\"$literal\": \(json)}" + } + if !plainSets.isEmpty { + stages.append("{\"$set\": {\(plainSets.joined(separator: ", "))}}") + } + let plainRemovals = writes.compactMap { write -> String? in + guard case .remove(let field) = write, !needsFieldExpression(field) else { return nil } + return MongoDocumentText.quoted(field) + } + if !plainRemovals.isEmpty { + stages.append("{\"$unset\": [\(plainRemovals.joined(separator: ", "))]}") + } + for write in writes where needsFieldExpression(write.field) { + stages.append(fieldExpressionStage(write)) + } + return "[\(stages.joined(separator: ", "))]" + } + + private static func fieldExpressionStage(_ write: FieldWrite) -> String { + let name = "\"field\": {\"$literal\": \(MongoDocumentText.quoted(write.field))}, \"input\": \"$$ROOT\"" + switch write { + case .set(_, let json): + return "{\"$replaceWith\": {\"$setField\": {\(name), \"value\": {\"$literal\": \(json)}}}}" + case .remove: + return "{\"$replaceWith\": {\"$unsetField\": {\(name)}}}" + } + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift b/Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift new file mode 100644 index 0000000000..2bb3a709eb --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift @@ -0,0 +1,84 @@ +// +// MongoDBWriteRefusal.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// A grid change the shell statement cannot carry as the value the user sees. +/// +/// Each one used to be left out of the statement, or written as something else, while the save +/// reported success. The generator throws it instead, and it reaches the user as the reason the +/// save was refused. +enum MongoDBWriteRefusal: Error, Equatable { + case binarySubtypeUnknown(field: String) + case binaryNeedsBytes(field: String) + case truncatedValue(field: String) + case unwritableFieldName(field: String) + case fieldNeedsMongoDB5(field: String) + case unreadableJSON(field: String) + case integerTooLarge(field: String) + case reorderedByTheShell(field: String) + case noDefaultValue(field: String) + case identityChanged + case missingIdentity + + var reason: String { + switch self { + case .binarySubtypeUnknown(let field): + return String( + format: String(localized: "The binary subtype of %@ is not known, so saving could store a different kind of binary. Change this field with a query."), + field + ) + case .binaryNeedsBytes(let field): + return String( + format: String(localized: "%@ holds binary data. Edit it as bytes, or set it to NULL."), + field + ) + case .truncatedValue(let field): + return String( + format: String(localized: "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query."), + field + ) + case .unwritableFieldName(let field): + return String( + format: String(localized: "The shell cannot write a field named \u{201C}%@\u{201D} into a new document. Insert this document with a query."), + field + ) + case .fieldNeedsMongoDB5(let field): + return String( + format: String(localized: "A field named \u{201C}%@\u{201D} can only be changed on MongoDB 5.0 or later, which can address a dot or a leading $ in a name."), + field + ) + case .unreadableJSON(let field): + return String( + format: String(localized: "%@ holds a document or an array, and this text is not valid JSON."), + field + ) + case .integerTooLarge(let field): + return String( + format: String(localized: "A number in %@ is larger than a 64-bit integer. Write it as a decimal, such as {\"$numberDecimal\": \"12345678901234567890\"}."), + field + ) + case .reorderedByTheShell(let field): + return String( + format: String(localized: "The shell would reorder or drop a key inside %@: it lists number-like keys first and drops __proto__. Change this field with a query."), + field + ) + case .noDefaultValue(let field): + return String( + format: String(localized: "MongoDB has no default values, so %@ cannot be set to DEFAULT."), + field + ) + case .identityChanged: + return String(localized: "MongoDB does not let a document's _id change. Duplicate the row with the new _id, then delete this one.") + case .missingIdentity: + return String(localized: "This row has no _id, so TablePro cannot tell which document to change.") + } + } + + func refusal(ofRow rowIndex: Int) -> PluginRowWriteRefusal { + PluginRowWriteRefusal(rowIndex: rowIndex, reason: reason) + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDocumentText.swift b/Plugins/MongoDBDriverPlugin/MongoDocumentText.swift index eab4400cbb..b0e7c934ab 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDocumentText.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDocumentText.swift @@ -91,6 +91,18 @@ struct MongoDocumentText: Equatable, Sendable { } extension MongoDocumentText.Value { + /// One value read with the same strictness and depth limit as a document, and with no rule + /// about its field names: a stored document may hold a top-level name that starts with `$`. + init(parsing text: String) throws { + var reader = MongoDocumentText.Reader(text) + reader.skipWhitespace() + guard !reader.isAtEnd else { throw MongoDocumentText.Refusal.empty } + let value = try reader.readValue(depth: 1) + reader.skipWhitespace() + guard reader.isAtEnd else { throw MongoDocumentText.Refusal.trailingContent } + self = value + } + var compactText: String { switch self { case .object(let members): diff --git a/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift b/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift new file mode 100644 index 0000000000..6a4a7023f2 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift @@ -0,0 +1,206 @@ +// +// MongoExtendedJsonForm.swift +// MongoDBDriverPlugin +// + +import Foundation +import os + +/// The two spellings of a nested document or array: the one a grid cell shows, and the one a write +/// hands the shell. +/// +/// A cell used to show the value rebuilt from Swift dictionaries, which had sorted its keys and +/// turned an ObjectId, a date, a decimal and a 64-bit integer into text or a bare number. Editing it +/// wrote those back as strings and doubles. The cell now shows the stored document's own Extended +/// JSON, in stored order, relaxed where that loses nothing: an int32 is a bare number, a double +/// always has a point or an exponent, a date inside 1970 to 9999 is ISO text with milliseconds, and +/// every other type keeps its wrapper. `$numberLong` stays wrapped, because the relaxed form of a +/// small 64-bit integer reads back as an int32. +enum MongoExtendedJsonForm { + typealias Value = MongoDocumentText.Value + typealias Member = MongoDocumentText.Member + + static let wrapperKeys: Set = [ + "$oid", "$symbol", "$numberInt", "$numberLong", "$numberDouble", "$numberDecimal", "$binary", + "$code", "$timestamp", "$regularExpression", "$dbPointer", "$date", "$minKey", "$maxKey", + "$undefined", "$uuid", "$regex" + ] + + /// An object that stands for one BSON value rather than an embedded document. + static func isWrapper(_ members: [Member]) -> Bool { + guard let first = members.first else { return false } + return wrapperKeys.contains(first.key) + } + + // MARK: - Display + + static func display(_ canonical: Value) -> Value { + switch canonical { + case .object(let members): + if let relaxed = relaxedScalar(members) { return relaxed } + if isWrapper(members) { return canonical } + return .object(members.map { Member(key: $0.key, value: display($0.value)) }) + case .array(let elements): + return .array(elements.map(display)) + case .string, .number, .literal: + return canonical + } + } + + private static func relaxedScalar(_ members: [Member]) -> Value? { + guard members.count == 1, let only = members.first else { return nil } + switch (only.key, only.value) { + case ("$numberInt", .string(let digits)): + return Int32(digits).map { .number(String($0)) } + case ("$numberDouble", .string(let text)): + return doubleText(text).map { .number($0) } + case ("$date", .object(let inner)): + return isoDateText(inner).map { .object([Member(key: "$date", value: .string($0))]) } + default: + return nil + } + } + + /// libbson writes a double with twenty significant digits, so 0.1 arrives as + /// 0.10000000000000000555. The shortest text that reads back as the same double is shown, with a + /// point or an exponent so it cannot be read back as an integer. + private static func doubleText(_ canonical: String) -> String? { + guard let value = Double(canonical), value.isFinite else { return nil } + let shortest = value.description + let isFloatingForm = shortest.contains(".") || shortest.contains("e") || shortest.contains("E") + return isFloatingForm ? shortest : shortest + ".0" + } + + /// The relaxed spec's range for an ISO date, and the only one `Date` formats without a sign. + private static let isoDateMilliseconds: ClosedRange = 0 ... 253_402_300_799_999 + + private static let isoFormatter = OSAllocatedUnfairLock(uncheckedState: { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + return formatter + }()) + + private static func isoDateText(_ inner: [Member]) -> String? { + guard inner.count == 1, let only = inner.first, only.key == "$numberLong", + case .string(let digits) = only.value, + let milliseconds = Int64(digits), isoDateMilliseconds.contains(milliseconds) else { + return nil + } + let seconds = Date(timeIntervalSince1970: TimeInterval(milliseconds / 1_000)) + let whole = isoFormatter.withLockUnchecked { $0.string(from: seconds) } + guard whole.hasSuffix("Z") else { return nil } + let fraction = String(format: ".%03lldZ", milliseconds % 1_000) + return String(whole.dropLast()) + fraction + } + + // MARK: - Shell + + /// The value with every number spelled so JavaScript cannot retype it, refused where the shell + /// would store something other than what the text shows. + /// + /// A whole-number double such as 3.0 is a JavaScript integer, which the shell sends as an int32, + /// and an integer past 2^53 is rounded, so both are wrapped. Other numbers stay bare and reach + /// the server as the same int32, int64 or double. An object is a JavaScript object, which lists + /// number-like keys first and reads `__proto__` as its prototype, so an object whose keys would + /// not survive in the order written is refused rather than stored in another order. + static func shellValue(_ display: Value, field: String) throws -> Value { + switch display { + case .number(let text): + return try shellNumber(text, field: field) + case .object(let members): + if isWrapper(members) { return try shellWrapper(members, field: field) } + guard MongoShellKeyOrder.survives(members.map(\.key)) else { + throw MongoDBWriteRefusal.reorderedByTheShell(field: field) + } + return .object(try members.map { Member(key: $0.key, value: try shellValue($0.value, field: field)) }) + case .array(let elements): + return .array(try elements.map { try shellValue($0, field: field) }) + case .string, .literal: + return display + } + } + + /// A wrapper as the shell has to be handed it to serialize it back unchanged. + /// + /// The shell's serializer spells every JavaScript number as a wrapper of its own, which libbson + /// then refuses inside a timestamp or a boundary key, where it reads a bare number. Those are + /// handed over as the shell's own `Timestamp`, `MinKey` and `MaxKey`, which serialize with + /// their numbers bare. A `$scope` is a document, so it is spelled like any other. Every other + /// wrapper holds only strings, which reach libbson as written. + private static func shellWrapper(_ members: [Member], field: String) throws -> Value { + if let expression = shellExpression(members) { + return .literal(expression) + } + return .object(try members.map { member in + guard member.key == "$scope" else { return member } + return Member(key: member.key, value: try shellValue(member.value, field: field)) + }) + } + + private static func shellExpression(_ members: [Member]) -> String? { + guard members.count == 1, let only = members.first else { return nil } + switch (only.key, only.value) { + case ("$minKey", .number("1")): + return "MinKey" + case ("$maxKey", .number("1")): + return "MaxKey" + case ("$timestamp", .object(let parts)): + let numbers = Dictionary(parts.map { ($0.key, $0.value) }, uniquingKeysWith: { first, _ in first }) + guard parts.count == 2, case .number(let seconds) = numbers["t"], case .number(let increment) = numbers["i"], + let time = UInt32(seconds), let ordinal = UInt32(increment) else { + return nil + } + return "Timestamp(\(time), \(ordinal))" + default: + return nil + } + } + + private static let largestExactInteger: UInt64 = 1 << 53 + + private static func shellNumber(_ text: String, field: String) throws -> Value { + let isFloatingForm = text.contains(".") || text.contains("e") || text.contains("E") + if isFloatingForm { + guard let value = Double(text), value.isFinite else { + throw MongoDBWriteRefusal.unreadableJSON(field: field) + } + let staysDouble = value.rounded(.towardZero) != value + return staysDouble ? .number(text) : wrapped("$numberDouble", text) + } + guard let integer = Int64(text) else { throw MongoDBWriteRefusal.integerTooLarge(field: field) } + return integer.magnitude > largestExactInteger ? wrapped("$numberLong", text) : .number(text) + } + + private static func wrapped(_ key: String, _ text: String) -> Value { + .object([Member(key: key, value: .string(text))]) + } +} + +/// Whether a JavaScript object keeps its keys in the order they were written. +/// +/// JavaScript lists array-index keys first, in ascending order, and treats `__proto__` as the +/// prototype rather than as a key, so neither kind of name reaches the server where it was written. +enum MongoShellKeyOrder { + static func survives(_ keys: [String]) -> Bool { + guard !keys.contains("__proto__") else { return false } + let moved = keys.filter(MongoDBCollectionDDL.isReorderedByTheShell) + guard !moved.isEmpty else { return true } + let shellOrder = moved.sorted(by: isNumericallyBefore) + keys.filter { !MongoDBCollectionDDL.isReorderedByTheShell($0) } + return shellOrder == keys + } + + private static func isNumericallyBefore(_ lhs: String, _ rhs: String) -> Bool { + let lhsLength = (lhs as NSString).length + let rhsLength = (rhs as NSString).length + return lhsLength == rhsLength ? lhs < rhs : lhsLength < rhsLength + } +} + +extension MongoDocumentText.Value { + var isContainer: Bool { + switch self { + case .object, .array: return true + case .string, .number, .literal: return false + } + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoNestedValueDiff.swift b/Plugins/MongoDBDriverPlugin/MongoNestedValueDiff.swift new file mode 100644 index 0000000000..2af644ab6f --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoNestedValueDiff.swift @@ -0,0 +1,75 @@ +// +// MongoNestedValueDiff.swift +// MongoDBDriverPlugin +// + +import Foundation + +/// The field paths an edit of a nested document or array changed, so a save writes those and +/// leaves every other embedded value as the server holds it. +/// +/// Writing the whole value back would send every untouched key through JavaScript, which lists +/// number-like keys first, and would overwrite what another client changed beside the edit. A path +/// is only used where the server applies it exactly: a key that cannot be addressed by a path, an +/// array whose length changed, a value whose type changed, or keys the server would append in an +/// order the edit did not write all fall back to writing the nearest enclosing value whole. +enum MongoNestedValueDiff { + typealias Value = MongoDocumentText.Value + typealias Member = MongoDocumentText.Member + + enum Change: Equatable { + case set(path: String, value: Value) + case remove(path: String) + } + + static func changes(from old: Value, to new: Value, at path: String) -> [Change] { + guard old != new else { return [] } + switch (old, new) { + case (.object(let oldMembers), .object(let newMembers)): + guard !MongoExtendedJsonForm.isWrapper(oldMembers), !MongoExtendedJsonForm.isWrapper(newMembers), + let changes = memberChanges(from: oldMembers, to: newMembers, at: path) else { + return [.set(path: path, value: new)] + } + return changes + case (.array(let oldElements), .array(let newElements)) where oldElements.count == newElements.count: + return zip(oldElements, newElements).enumerated().flatMap { index, pair in + changes(from: pair.0, to: pair.1, at: "\(path).\(index)") + } + default: + return [.set(path: path, value: new)] + } + } + + /// Nil when the edit cannot be expressed key by key, which writes the object whole. + /// + /// The server keeps an existing key where it is and appends a new one at the end, and several + /// new keys in one update land in its own order rather than the one written, so more than one + /// new key is written as the whole object. + private static func memberChanges(from old: [Member], to new: [Member], at path: String) -> [Change]? { + let oldKeys = old.map(\.key) + let newKeys = new.map(\.key) + let newKeySet = Set(newKeys) + let oldKeySet = Set(oldKeys) + let added = newKeys.filter { !oldKeySet.contains($0) } + guard added.count <= 1 else { return nil } + guard oldKeys.filter(newKeySet.contains) + added == newKeys else { return nil } + + var changes: [Change] = [] + for key in oldKeys where !newKeySet.contains(key) { + guard BsonDocumentFlattener.isAddressableSegment(key) else { return nil } + changes.append(.remove(path: "\(path).\(key)")) + } + for member in new { + let oldValue = old.first { $0.key == member.key }?.value + guard oldValue != member.value else { continue } + guard BsonDocumentFlattener.isAddressableSegment(member.key) else { return nil } + let memberPath = "\(path).\(member.key)" + guard let oldValue else { + changes.append(.set(path: memberPath, value: member.value)) + continue + } + changes += Self.changes(from: oldValue, to: member.value, at: memberPath) + } + return changes + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptResultBuilder.swift b/Plugins/MongoDBDriverPlugin/MongoScriptResultBuilder.swift index 274b1960e8..4a9dedc163 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptResultBuilder.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptResultBuilder.swift @@ -11,7 +11,7 @@ enum MongoScriptResultBuilder { for outcome: MongoScriptStatementResult, startTime: Date, emptyColumns: [(name: String, typeName: String)] = [(name: "_id", typeName: "ObjectId")], - documents build: ([[String: Any]], String, Bool) -> PluginQueryResult + documents build: (MongoReadDocuments, String, Bool) -> PluginQueryResult ) -> PluginQueryResult { if outcome.producedDocuments, outcome.documents.json.isEmpty { // Zero columns reads as write-success in the result pane, so a query that matched @@ -25,7 +25,7 @@ enum MongoScriptResultBuilder { if outcome.producedDocuments { let grid = build( - outcome.documents.dictionaries, + outcome.documents.readDocuments, outcome.collection ?? "", outcome.documents.isTruncated ).withRowsAffected(outcome.rowsAffected) diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 0482604dcf..54e76d9ee2 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -184447,6 +184447,39 @@ }, "TTL has to be a whole number of seconds above 0, or -1 or NULL for no expiry." : { + }, + "%@ holds binary data. Edit it as bytes, or set it to NULL." : { + + }, + "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query." : { + + }, + "The shell cannot write a field named “%@” into a new document. Insert this document with a query." : { + + }, + "%@ holds a document or an array, and this text is not valid JSON." : { + + }, + "A number in %@ is larger than a 64-bit integer. Write it as a decimal, such as {\"$numberDecimal\": \"12345678901234567890\"}." : { + + }, + "MongoDB has no default values, so %@ cannot be set to DEFAULT." : { + + }, + "MongoDB does not let a document's _id change. Duplicate the row with the new _id, then delete this one." : { + + }, + "This row has no _id, so TablePro cannot tell which document to change." : { + + }, + "The binary subtype of %@ is not known, so saving could store a different kind of binary. Change this field with a query." : { + + }, + "A field named “%@” can only be changed on MongoDB 5.0 or later, which can address a dot or a leading $ in a name." : { + + }, + "The shell would reorder or drop a key inside %@: it lists number-like keys first and drops __proto__. Change this field with a query." : { + } }, "version" : "1.1" diff --git a/TableProTests/Plugins/MongoDBCollectionDDLTests.swift b/TableProTests/Plugins/MongoDBCollectionDDLTests.swift index 39b91b0219..76c73fd99e 100644 --- a/TableProTests/Plugins/MongoDBCollectionDDLTests.swift +++ b/TableProTests/Plugins/MongoDBCollectionDDLTests.swift @@ -188,12 +188,12 @@ struct MongoDBCollectionDDLTests { #expect(MongoDBCollectionDDL.refusal(for: .addColumn(column("f", alias))) == nil) } - @Test("Field names a JavaScript object reorders or drops are refused", arguments: ["10", "0", "__proto__"]) + @Test("Field names a JavaScript object reorders or drops are refused", arguments: ["10", "0", "4294967294", "__proto__"]) func shellReorderedNamesAreRefused(name: String) { #expect(MongoDBCollectionDDL.refusal(for: .addColumn(column(name, "string"))) != nil) } - @Test("Names that only look numeric are kept", arguments: ["007", "10a", "a10"]) + @Test("Names that only look numeric, or are past the array-index range, are kept", arguments: ["007", "10a", "a10", "4294967295", "99999999999"]) func nonCanonicalNumericNamesAreKept(name: String) { #expect(MongoDBCollectionDDL.refusal(for: .addColumn(column(name, "string"))) == nil) } diff --git a/TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift b/TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift new file mode 100644 index 0000000000..0966e41ac3 --- /dev/null +++ b/TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift @@ -0,0 +1,209 @@ +// +// MongoDBFieldNameUpdateTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +/// Measured on MongoDB 7.0.43: a classic `$set` reads each key as a path, so `price.usd` walks into +/// a sub-document, `$price` is refused with code 52 and the empty name with code 56. Inside a +/// pipeline, `$set` of `tags.1` rewrote every element of `tags` and reported success. +struct MongoDBFieldNameUpdateTests { + private func update( + _ cells: [(column: String, old: PluginCellValue, new: PluginCellValue)], + columns: [String], + kinds: [String: BsonValueKind] = [:], + version: String? = "7.0.43" + ) throws -> String { + let gen = MongoDBStatementGenerator( + collectionName: "items", + columns: columns, + columnKinds: kinds, + capabilities: { MongoDBCapabilities.parse(version) } + ) + let change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: cells.map { cell in + (columnIndex: columns.firstIndex(of: cell.column) ?? 0, columnName: cell.column, oldValue: cell.old, newValue: cell.new) + }, + originalRow: [.text("1")] + columns.dropFirst().map { _ in PluginCellValue.null } + ) + let writes = try gen.generateRowWrites( + from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + ) + return try #require(writes.first?.statement) + } + + /// The update argument of `updateOne(filter, update)`, parsed. + private func updateArgument(of statement: String) throws -> Any { + let open = try #require(statement.range(of: "}, ")).upperBound + let text = String(statement[open ..< statement.index(before: statement.endIndex)]) + return try JSONSerialization.jsonObject(with: Data(text.utf8)) + } + + private func stages(of statement: String) throws -> [[String: Any]] { + try #require(try updateArgument(of: statement) as? [[String: Any]]) + } + + @Test("Editing a field named price.usd sends $setField with $literal, not $set on a path") + func dottedNameUsesSetField() throws { + let statement = try update([("price.usd", "10", "12")], columns: ["_id", "price.usd"], kinds: ["price.usd": .int32]) + + let setField = #"{"field": {"$literal": "price.usd"}, "input": "$$ROOT", "value": {"$literal": 12}}"# + #expect(statement == #"db.items.updateOne({"_id": 1}, [{"$replaceWith": {"$setField": "# + setField + "}}])") + } + + @Test("A leading $, the empty name and __proto__ take the same route") + func otherUnaddressableNamesUseSetField() throws { + for name in ["$price", "", "__proto__"] { + let statement = try update([(name, "a", "b")], columns: ["_id", name]) + let stage = try #require(try stages(of: statement).first) + let setField = try #require((stage["$replaceWith"] as? [String: Any])?["$setField"] as? [String: Any]) + #expect((setField["field"] as? [String: Any])?["$literal"] as? String == name, "\(name)") + #expect((setField["value"] as? [String: Any])?["$literal"] as? String == "b", "\(name)") + } + } + + @Test("Removing a field with a dotted name uses $unsetField") + func dottedNameRemovalUsesUnsetField() throws { + let statement = try update([("price.usd", "10", nil)], columns: ["_id", "price.usd"]) + + #expect(statement.contains(#"{"$replaceWith": {"$unsetField": {"field": {"$literal": "price.usd"}, "input": "$$ROOT"}}}"#)) + #expect(!statement.contains(#""$unset": {"#)) + } + + @Test("A row mixing an ordinary and a dotted field is one pipeline: $set with $literal, then $replaceWith") + func mixedRowIsOnePipeline() throws { + let statement = try update( + [("name", "a", "b"), ("price.usd", "10", "12"), ("note", "x", nil)], + columns: ["_id", "name", "price.usd", "note"] + ) + + let stages = try stages(of: statement) + #expect(stages.count == 3) + #expect(((stages[0]["$set"] as? [String: Any])?["name"] as? [String: Any])?["$literal"] as? String == "b") + #expect(stages[1]["$unset"] as? [String] == ["note"]) + #expect(stages[2]["$replaceWith"] != nil) + } + + @Test("A row of ordinary fields keeps the classic $set document, which every server version reads") + func ordinaryRowStaysClassic() throws { + let statement = try update([("name", "a", "b"), ("note", "x", nil)], columns: ["_id", "name", "note"], version: "4.4.0") + + #expect(statement == #"db.items.updateOne({"_id": 1}, {"$set": {"name": "b"}, "$unset": {"note": ""}})"#) + } + + @Test("A typed value inside $literal keeps its wrapper") + func typedValueKeepsItsWrapper() throws { + let statement = try update( + [("when", "2024-01-02T03:04:05Z", "2024-01-02T03:04:06Z"), ("a.b", "1", "2")], + columns: ["_id", "when", "a.b"], + kinds: ["when": .date] + ) + + #expect(statement.contains(#""when": {"$literal": {"$date": {"$numberLong": "1704164646000"}}}"#)) + } + + @Test("A string that looks like a field path is wrapped in $literal so the pipeline does not read it") + func pathLikeStringIsLiteral() throws { + let statement = try update([("name", "a", "$notAPath"), ("a.b", "1", "2")], columns: ["_id", "name", "a.b"]) + + #expect(statement.contains(#""name": {"$literal": "$notAPath"}"#)) + } + + @Test("A server older than 5.0 refuses before anything is sent, and an unknown version sends the pipeline") + func pipelineNeedsMongoDB5() throws { + #expect(throws: MongoDBWriteRefusal.fieldNeedsMongoDB5(field: "price.usd").refusal(ofRow: 0)) { + try update([("price.usd", "10", "12")], columns: ["_id", "price.usd"], version: "4.4.29") + } + let unknown = try update([("price.usd", "10", "12")], columns: ["_id", "price.usd"], version: nil) + #expect(unknown.contains("$setField")) + } + + @Test("$setField is known from 5.0 on, and unknown when the version is") + func fieldExpressionCapability() { + #expect(MongoDBCapabilities.parse("4.4.29").supportsFieldExpressions == false) + #expect(MongoDBCapabilities.parse("5.0.0").supportsFieldExpressions == true) + #expect(MongoDBCapabilities.parse("7.0.43").supportsFieldExpressions == true) + #expect(MongoDBCapabilities.parse(nil).supportsFieldExpressions == nil) + } + + // MARK: - Pipeline and nested paths together + + @Test("A row that edits price.usd and tags[1] writes the whole tags array, and no stage names a path") + func pipelineWritesArraysWhole() throws { + let statement = try update( + [("price.usd", "10", "12"), ("tags", #"["a","b","c"]"#, #"["a","Z","c"]"#)], + columns: ["_id", "price.usd", "tags"], + kinds: ["tags": .array] + ) + + let stages = try stages(of: statement) + let set = try #require(stages.first?["$set"] as? [String: Any]) + #expect(set.keys.allSatisfy { !$0.contains(".") }) + #expect((set["tags"] as? [String: Any])?["$literal"] as? [String] == ["a", "Z", "c"]) + #expect(!statement.contains("tags.1")) + } + + @Test("A removal inside an array of documents is written as the whole array under the pipeline route") + func pipelineWritesNestedRemovalWhole() throws { + let statement = try update( + [("$p", "1", "2"), ("items", #"[{"sku":"a","qty":1}]"#, #"[{"sku":"a"}]"#)], + columns: ["_id", "$p", "items"], + kinds: ["items": .array] + ) + + let stages = try stages(of: statement) + let set = try #require(stages.first?["$set"] as? [String: Any]) + let items = try #require((set["items"] as? [String: Any])?["$literal"] as? [[String: Any]]) + #expect(items.count == 1) + #expect(items[0]["qty"] == nil) + #expect(!statement.contains("items.0")) + #expect(stages.allSatisfy { $0["$unset"] == nil }) + } + + @Test("The same edits without a special name stay classic and name only the changed paths") + func classicRouteKeepsPaths() throws { + let statement = try update( + [("tags", #"["a","b","c"]"#, #"["a","Z","c"]"#), ("items", #"[{"sku":"a","qty":1}]"#, #"[{"sku":"a"}]"#)], + columns: ["_id", "tags", "items"], + kinds: ["tags": .array, "items": .array] + ) + + #expect(statement == #"db.items.updateOne({"_id": 1}, {"$set": {"tags.1": "Z"}, "$unset": {"items.0.qty": ""}})"#) + } + + // MARK: - Inserts + + @Test("A new document cannot hold __proto__ or an empty name, which the shell cannot write") + func insertRefusesUnwritableNames() { + for name in ["__proto__", ""] { + let gen = MongoDBStatementGenerator(collectionName: "items", columns: ["_id", name]) + #expect(throws: MongoDBWriteRefusal.unwritableFieldName(field: name).refusal(ofRow: 0), "\(name)") { + try gen.generateRowWrites( + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + insertedRowData: [0: [nil, "x"]], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + } + } + } + + @Test("A new document keeps a dotted or $ name as a literal key, which insert accepts") + func insertKeepsDottedNames() throws { + let gen = MongoDBStatementGenerator(collectionName: "items", columns: ["_id", "k.d", "$q"]) + + let writes = try gen.generateRowWrites( + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + insertedRowData: [0: [nil, "7", "8"]], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + + #expect(writes.first?.statement == #"db.items.insertOne({"k.d": 7, "$q": 8})"#) + } +} diff --git a/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift b/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift new file mode 100644 index 0000000000..832dc91d5b --- /dev/null +++ b/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift @@ -0,0 +1,344 @@ +// +// MongoDBNestedValueWriteTests.swift +// TableProTests +// + +import Foundation +import JavaScriptCore +import TableProNumberFormatting +import TableProPluginKit +import Testing + +struct MongoDBNestedValueWriteTests { + private static let address = #"{"zip":"100","city":"HN","verifiedAt":{"$date":"2024-05-01T10:00:00.123Z"},"# + + #""ownerId":{"$oid":"65a1b2c3d4e5f60718293a4b"},"n":{"$numberLong":"5"},"d":3.0}"# + + private func update( + _ column: String, + from oldText: String, + to newText: String, + kind: BsonValueKind = .document + ) throws -> String { + let gen = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", column], columnKinds: [column: kind]) + let change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: 1, columnName: column, oldValue: .text(oldText), newValue: .text(newText))], + originalRow: [.text("1"), .text(oldText)] + ) + let writes = try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []) + return try #require(writes.first?.statement) + } + + private func insert(_ column: String, _ text: String) throws -> String { + let gen = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", column]) + let writes = try gen.generateRowWrites( + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + insertedRowData: [0: [nil, .text(text)]], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + return try #require(writes.first?.statement) + } + + // MARK: - Display + + @Test("A nested document shows its stored order and its BSON types") + func displayKeepsOrderAndTypes() throws { + let canonical = #"{"_id":{"$numberInt":"1"},"address":{"zip":"100","city":"HN","# + + #""verifiedAt":{"$date":{"$numberLong":"1714557600123"}},"ownerId":{"$oid":"65a1b2c3d4e5f60718293a4b"},"# + + #""amount":{"$numberDecimal":"1847.270000000000000000000000001"},"n":{"$numberLong":"5"},"# + + #""d":{"$numberDouble":"3.0"},"i":{"$numberInt":"7"},"bin":{"$binary":{"base64":"AAECAw==","subType":"05"}}}}"# + let document: [String: Any] = ["_id": Int32(1), "address": ["zip": "100"]] + + let rows = BsonDocumentFlattener.flatten( + documents: [document], columns: ["_id", "address"], kinds: [.int32, .document], + representation: .unspecified, storedTexts: [canonical] + ) + + let shown = #"{"zip":"100","city":"HN","verifiedAt":{"$date":"2024-05-01T10:00:00.123Z"},"# + + #""ownerId":{"$oid":"65a1b2c3d4e5f60718293a4b"},"amount":{"$numberDecimal":"1847.270000000000000000000000001"},"# + + #""n":{"$numberLong":"5"},"d":3.0,"i":7,"bin":{"$binary":{"base64":"AAECAw==","subType":"05"}}}"# + #expect(rows[0][1] == .text(shown)) + } + + @Test("Doubles show their shortest spelling, and values outside the relaxed form keep their wrappers") + func displayKeepsWrappersItCannotRelax() { + let canonical = MongoDocumentText.Value.object([ + .init(key: "tenth", value: .object([.init(key: "$numberDouble", value: .string("0.10000000000000000555"))])), + .init(key: "e", value: .object([.init(key: "$numberDouble", value: .string("1e+20"))])), + .init(key: "nan", value: .object([.init(key: "$numberDouble", value: .string("NaN"))])), + .init(key: "old", value: .object([.init(key: "$date", value: .object([.init(key: "$numberLong", value: .string("-1"))]))])) + ]) + + let shown = MongoExtendedJsonForm.display(canonical).compactText + + #expect(shown == #"{"tenth":0.1,"e":1e+20,"nan":{"$numberDouble":"NaN"},"old":{"$date":{"$numberLong":"-1"}}}"#) + } + + @Test("A cell without stored text keeps the older rendering") + func displayWithoutStoredTextIsUnchanged() { + let rows = BsonDocumentFlattener.flatten( + documents: [["tags": ["b", "a"]]], columns: ["tags"], kinds: [.array], representation: .unspecified + ) + + #expect(rows[0][0] == .text(#"["b","a"]"#)) + } + + // MARK: - Updates + + @Test("Editing one nested key writes only that path") + func singleLeafEdit() throws { + let edited = Self.address.replacingOccurrences(of: #""city":"HN""#, with: #""city":"HN2""#) + + let statement = try update("address", from: Self.address, to: edited) + + #expect(statement == #"db.people.updateOne({"_id": 1}, {"$set": {"address.city": "HN2"}})"#) + } + + @Test("Removing a nested key writes $unset on its path") + func nestedRemoval() throws { + let edited = Self.address.replacingOccurrences(of: #""zip":"100","#, with: "") + + let statement = try update("address", from: Self.address, to: edited) + + #expect(statement == #"db.people.updateOne({"_id": 1}, {"$unset": {"address.zip": ""}})"#) + } + + @Test("One key added at the end is set on its path") + func appendedKeyIsAPath() throws { + let edited = String(Self.address.dropLast()) + #","extra":{"$numberLong":"9"}}"# + + let statement = try update("address", from: Self.address, to: edited) + + #expect(statement == #"db.people.updateOne({"_id": 1}, {"$set": {"address.extra": {"$numberLong":"9"}}})"#) + } + + @Test("Keys the server would append in its own order are written as the whole object") + func reorderedAdditionsAreWrittenWhole() throws { + let twoNew = try update("meta", from: #"{"a":1}"#, to: #"{"a":1,"z":2,"b":3}"#) + let inserted = try update("meta", from: #"{"a":1,"c":3}"#, to: #"{"a":1,"b":2,"c":3}"#) + let moved = try update("meta", from: #"{"a":1,"b":2}"#, to: #"{"b":2,"a":1}"#) + + #expect(twoNew.contains(#"{"$set": {"meta": {"a":1,"z":2,"b":3}}}"#)) + #expect(inserted.contains(#"{"$set": {"meta": {"a":1,"b":2,"c":3}}}"#)) + #expect(moved.contains(#"{"$set": {"meta": {"b":2,"a":1}}}"#)) + } + + @Test("A same-length array edit writes the index, a length change writes the array with its wrappers") + func arrayEdits() throws { + let tags = #"[{"$oid":"65a1b2c3d4e5f60718293a4b"},"b",3.0]"# + + let index = try update("tags", from: tags, to: tags.replacingOccurrences(of: #""b""#, with: #""B""#), kind: .array) + let grown = try update("tags", from: tags, to: String(tags.dropLast()) + #","c"]"#, kind: .array) + + #expect(index == #"db.people.updateOne({"_id": 1}, {"$set": {"tags.1": "B"}})"#) + #expect(grown == #"db.people.updateOne({"_id": 1}, {"$set": {"tags": [{"$oid":"65a1b2c3d4e5f60718293a4b"},"b",{"$numberDouble":"3.0"},"c"]}})"#) + } + + @Test("A changed nested key that holds a dot is written as the whole enclosing value") + func unaddressableNestedKeyEscalates() throws { + let statement = try update("meta", from: #"{"a.b":1,"c":2}"#, to: #"{"a.b":5,"c":2}"#) + + #expect(statement.contains(#"{"$set": {"meta": {"a.b":5,"c":2}}}"#)) + } + + @Test("A value whose type changed is written whole at its path") + func typeChangeWritesTheValue() throws { + let statement = try update("meta", from: #"{"a":{"x":1}}"#, to: #"{"a":[1]}"#) + + #expect(statement.contains(#"{"$set": {"meta.a": [1]}}"#)) + } + + @Test("An edit that only reformats the text still writes the value, so the row is not left unwritten") + func reformatOnlyWritesTheWholeValue() throws { + let statement = try update("meta", from: #"{"a":1}"#, to: "{\n \"a\": 1\n}") + + #expect(statement.contains(#"{"$set": {"meta": {"a":1}}}"#)) + } + + @Test("Text that is not JSON in a document column is refused") + func unreadableJsonIsRefused() { + #expect(throws: MongoDBWriteRefusal.unreadableJSON(field: "meta").refusal(ofRow: 0)) { + try update("meta", from: #"{"a":1}"#, to: #"{"a":}"#) + } + } + + @Test("A complete value typed over one shortened for display replaces the field whole") + func truncatedOldValueIsReplacedWhole() throws { + let stored = #"{"big":""# + String(repeating: "x", count: 12_000) + #"","keep":1}"# + let shown = JSONTruncation.truncate(stored, maxLength: BsonDocumentFlattener.maxNestedJsonLength) + + let statement = try update("meta", from: shown, to: #"{"a":1,"d":3.0}"#) + + #expect(statement == #"db.people.updateOne({"_id": 1}, {"$set": {"meta": {"a":1,"d":{"$numberDouble":"3.0"}}}})"#) + } + + @Test("An edit that keeps the shortened text is still refused") + func truncatedEditIsRefused() { + let stored = #"{"keep":1,"big":""# + String(repeating: "x", count: 12_000) + #""}"# + let shown = JSONTruncation.truncate(stored, maxLength: BsonDocumentFlattener.maxNestedJsonLength) + + #expect(throws: MongoDBWriteRefusal.truncatedValue(field: "meta").refusal(ofRow: 0)) { + try update("meta", from: shown, to: shown.replacingOccurrences(of: #""keep":1"#, with: #""keep":2"#)) + } + } + + // MARK: - Whole values + + @Test("A whole value keeps a whole-number double as a double and a 64-bit integer as one") + func wholeValueKeepsNumberTypes() throws { + let statement = try insert("meta", #"{"d":3.0,"z":-0.0,"e":1e3,"f":1.5,"n":{"$numberLong":"5"},"big":9007199254740993,"i":7}"#) + + let written = #""meta": {"d":{"$numberDouble":"3.0"},"z":{"$numberDouble":"-0.0"},"e":{"$numberDouble":"1e3"},"# + + #""f":1.5,"n":{"$numberLong":"5"},"big":{"$numberLong":"9007199254740993"},"i":7}"# + #expect(statement.contains(written)) + } + + @Test("An integer past 64 bits inside a value is refused") + func integerPastInt64IsRefused() { + #expect(throws: MongoDBWriteRefusal.integerTooLarge(field: "meta").refusal(ofRow: 0)) { + try insert("meta", #"{"n":99999999999999999999}"#) + } + } + + @Test("A key JavaScript would move or drop is refused, and number-like keys already first are written") + func shellKeyOrder() throws { + #expect(throws: MongoDBWriteRefusal.reorderedByTheShell(field: "meta").refusal(ofRow: 0)) { + try insert("meta", #"{"zip":"1","2":"two"}"#) + } + #expect(throws: MongoDBWriteRefusal.reorderedByTheShell(field: "meta").refusal(ofRow: 0)) { + try insert("meta", #"{"a":{"__proto__":1}}"#) + } + #expect(throws: MongoDBWriteRefusal.reorderedByTheShell(field: "meta").refusal(ofRow: 0)) { + try insert("meta", #"{"10":1,"9":2}"#) + } + let ordered = try insert("meta", #"{"2":"two","10":"ten","zip":"1"}"#) + #expect(ordered.contains(#""meta": {"2":"two","10":"ten","zip":"1"}"#)) + } + + @Test("Only array-index keys up to 4294967294 are moved, so a larger number-like key is written where it is") + func shellKeyOrderStopsAtTheArrayIndexRange() throws { + #expect(throws: MongoDBWriteRefusal.reorderedByTheShell(field: "meta").refusal(ofRow: 0)) { + try insert("meta", #"{"name":"a","4294967294":"b"}"#) + } + let past = try insert("meta", #"{"name":"a","4294967295":"b","99999999999":"c"}"#) + #expect(past.contains(#""meta": {"name":"a","4294967295":"b","99999999999":"c"}"#)) + } + + @Test("A timestamp and the boundary keys are written as the shell's own values, which keep their numbers bare") + func numericWrappersUseTheShellsValues() throws { + let statement = try insert( + "meta", + #"{"ts":{"$timestamp":{"t":1700000000,"i":7}},"lo":{"$minKey":1},"hi":{"$maxKey":1},"# + + #""arr":[{"$timestamp":{"i":2,"t":4294967295}},{"$maxKey":1}]}"# + ) + + #expect(statement.contains( + #""meta": {"ts":Timestamp(1700000000, 7),"lo":MinKey,"hi":MaxKey,"arr":[Timestamp(4294967295, 2),MaxKey]}"# + )) + } + + @Test("A code scope is spelled as a document, so its numbers keep their types and its keys their order") + func codeScopeIsSpelledAsADocument() throws { + let statement = try insert("meta", #"{"c":{"$code":"f","$scope":{"n":3.0,"i":{"$numberInt":"1"}}}}"#) + + #expect(statement.contains(#""meta": {"c":{"$code":"f","$scope":{"n":{"$numberDouble":"3.0"},"i":{"$numberInt":"1"}}}}"#)) + #expect(throws: MongoDBWriteRefusal.reorderedByTheShell(field: "meta").refusal(ofRow: 0)) { + try insert("meta", #"{"c":{"$code":"f","$scope":{"a":1,"2":2}}}"#) + } + } + + // MARK: - Through the shell + + private static let everyWrapper = #"{"oid":{"$oid":"65a1b2c3d4e5f60718293a4b"},"sym":{"$symbol":"s"},"# + + #""i":{"$numberInt":"7"},"l":{"$numberLong":"5"},"d":{"$numberDouble":"3.0"},"f":{"$numberDouble":"1.5"},"# + + #""dec":{"$numberDecimal":"1.5"},"bin":{"$binary":{"base64":"AAECAw==","subType":"05"}},"# + + #""code":{"$code":"f"},"scoped":{"$code":"f","$scope":{"x":{"$numberInt":"1"}}},"# + + #""ts":{"$timestamp":{"t":1700000000,"i":7}},"re":{"$regularExpression":{"pattern":"^a","options":"i"}},"# + + #""ptr":{"$dbPointer":{"$ref":"c","$id":{"$oid":"65a1b2c3d4e5f60718293a4b"}}},"# + + #""old":{"$date":{"$numberLong":"-1"}},"lo":{"$minKey":1},"hi":{"$maxKey":1},"u":{"$undefined":true},"# + + #""arr":[{"$timestamp":{"t":1,"i":2}},{"$minKey":1}],"# + + #""big":{"name":{"$numberInt":"1"},"4294967295":{"$numberInt":"2"}}}"# + + private func shownCell(_ canonical: String) throws -> PluginCellValue { + let rows = BsonDocumentFlattener.flatten( + documents: [["_id": Int32(1), "m": ["x": 1]]], columns: ["_id", "m"], kinds: [.int32, .document], + representation: .unspecified, storedTexts: [#"{"_id":{"$numberInt":"1"},"m":"# + canonical + "}"] + ) + return try #require(rows.first?[1]) + } + + private func evaluate(_ statement: String, reply: String) throws -> MongoScriptPreludeTests.RecordingHost { + let host = MongoScriptPreludeTests.RecordingHost() + host.replies = [reply] + let context = try MongoScriptContext.make(execute: { host.handle($0) }, emit: { host.record(printed: $0) }) + context.evaluateScript(statement) + #expect(context.exception == nil, "\(context.exception?.toString() ?? "")") + return host + } + + @Test("A duplicated row hands the shell every nested BSON type as the Extended JSON it was read as") + func duplicateRoundTripsThroughTheShell() throws { + let cell = try shownCell(Self.everyWrapper) + let gen = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", "m"], columnKinds: ["m": .document]) + let writes = try gen.generateRowWrites( + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + insertedRowData: [0: ["__DEFAULT__", cell]], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + let statement = try #require(writes.first?.statement) + + let host = try evaluate(statement, reply: #"{"insertedIds": [{"$oid": "507f1f77bcf86cd799439011"}], "insertedCount": 1}"#) + + #expect(host.requests(op: "insertOne").first?["document"] as? String == #"{"m":"# + Self.everyWrapper + "}") + } + + @Test("A nested value written whole hands the shell its BSON types unchanged") + func wholeRewriteRoundTripsThroughTheShell() throws { + let cell = try shownCell(Self.everyWrapper) + let shown = try #require(cell.asText) + let grown = String(shown.dropLast()) + #","extra":1,"extra2":2}"# + + let statement = try update("m", from: shown, to: grown) + let host = try evaluate(statement, reply: #"{"n": 1, "nModified": 1}"#) + + let written = String(Self.everyWrapper.dropLast()) + #","extra":{"$numberInt":"1"},"extra2":{"$numberInt":"2"}}"# + #expect(host.requests(op: "update").first?["update"] as? String == #"{"$set":{"m":"# + written + "}}") + } + + @Test("A duplicated row keeps a nested ObjectId and date") + func duplicateKeepsNestedWrappers() throws { + let statement = try insert("address", Self.address) + + #expect(statement.contains(#""ownerId":{"$oid":"65a1b2c3d4e5f60718293a4b"}"#)) + #expect(statement.contains(#""verifiedAt":{"$date":"2024-05-01T10:00:00.123Z"}"#)) + #expect(statement.contains(#""d":{"$numberDouble":"3.0"}"#)) + } + + @Test("A document _id is matched in the order it is stored") + func documentIdKeepsItsOrder() throws { + let gen = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", "name"], identityKind: .document) + let change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: 1, columnName: "name", oldValue: "a", newValue: "b")], + originalRow: [.text(#"{"z":1,"a":{"$numberLong":"2"}}"#), "a"] + ) + + let writes = try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []) + + #expect(writes.first?.statement == #"db.people.updateOne({"_id": {"z":1,"a":{"$numberLong":"2"}}}, {"$set": {"name": "b"}})"#) + } + + // MARK: - Parsing one value + + @Test("One value parses with the document reader's strictness, top-level $ names included") + func valueParsing() throws { + #expect(try MongoDocumentText.Value(parsing: #"{"$price":1}"#) == .object([.init(key: "$price", value: .number("1"))])) + #expect(try MongoDocumentText.Value(parsing: " [1, \"a\"] ") == .array([.number("1"), .string("a")])) + #expect(throws: MongoDocumentText.Refusal.trailingContent) { try MongoDocumentText.Value(parsing: "[1] [2]") } + #expect(throws: MongoDocumentText.Refusal.duplicateField("a")) { try MongoDocumentText.Value(parsing: #"{"a":1,"a":2}"#) } + } +} diff --git a/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift b/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift index b1149aa0b4..f51a112f43 100644 --- a/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift +++ b/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift @@ -13,7 +13,7 @@ struct MongoDBStatementGeneratorTests { // MARK: - INSERT @Test("Simple insert generates insertOne, skipping _id") - func simpleInsert() { + func simpleInsert() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name", "email"] @@ -30,7 +30,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, "Alice", "alice@example.com"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -46,7 +46,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Insert skips __DEFAULT__ sentinel values") - func insertSkipsDefaultSentinel() { + func insertSkipsDefaultSentinel() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name", "age"] @@ -63,7 +63,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, "Bob", "__DEFAULT__"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -78,7 +78,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Insert with nil values are excluded from document") - func insertNilValuesExcluded() { + func insertNilValuesExcluded() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name", "email"] @@ -95,7 +95,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, "Carol", nil] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -108,8 +108,8 @@ struct MongoDBStatementGeneratorTests { #expect(!stmt.contains("\"email\"")) } - @Test("Insert with all nil/default values produces no statement") - func insertAllNilProducesNothing() { + @Test("A new row with every cell empty inserts the empty document") + func insertAllNilWritesEmptyDocument() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -123,21 +123,22 @@ struct MongoDBStatementGeneratorTests { ) let insertedData: [Int: [PluginCellValue]] = [ - 0: [nil, nil] + 0: [nil, "__DEFAULT__"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], insertedRowIndices: [0] ) - #expect(results.isEmpty) + #expect(results.map { $0.statement } == ["db.users.insertOne({})"]) + #expect(results.map { $0.rowIndices } == [[0]]) } @Test("Insert uses cellChanges as fallback when insertedRowData missing") - func insertFallbackToCellChanges() { + func insertFallbackToCellChanges() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -152,7 +153,7 @@ struct MongoDBStatementGeneratorTests { originalRow: nil ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], @@ -164,7 +165,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Insert with numeric value auto-detects type") - func insertNumericValue() { + func insertNumericValue() throws { let gen = MongoDBStatementGenerator( collectionName: "data", columns: ["_id", "count"] @@ -181,7 +182,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, "42"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -193,7 +194,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Insert emits JSON-valid decimal and exponent numbers") - func insertEmitsJsonValidNumbers() { + func insertEmitsJsonValidNumbers() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "decimal", "exponent"] @@ -210,7 +211,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, "0.5", "1e3"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -223,7 +224,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Insert quotes non-JSON numeric spellings") - func insertQuotesNonJsonNumericSpellings() { + func insertQuotesNonJsonNumericSpellings() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "leadingDecimal", "trailingDecimal", "leadingPlus", "leadingZero"] @@ -240,7 +241,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, ".5", "1.", "+7", "01"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -257,7 +258,7 @@ struct MongoDBStatementGeneratorTests { /// The maximum Int64 is past 2^53, where a bare JavaScript literal rounds: JavaScriptCore reads /// `9223372036854775807` as `9223372036854776000`. It has to cross as `$numberLong`. @Test("Insert quotes integers that overflow Int64 and writes the largest ones as $numberLong") - func insertQuotesInt64Overflow() { + func insertQuotesInt64Overflow() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "overflow", "maxInt64"] @@ -274,7 +275,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, "12345678901234567890", "9223372036854775807"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -287,7 +288,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Insert not in insertedRowIndices is skipped") - func insertNotInIndicesSkipped() { + func insertNotInIndicesSkipped() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -304,7 +305,7 @@ struct MongoDBStatementGeneratorTests { 5: [nil, "Eve"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: insertedData, deletedRowIndices: [], @@ -317,7 +318,7 @@ struct MongoDBStatementGeneratorTests { // MARK: - UPDATE @Test("Update with ObjectId _id") - func updateWithObjectId() { + func updateWithObjectId() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name", "email"] @@ -333,7 +334,7 @@ struct MongoDBStatementGeneratorTests { originalRow: [.text(objectId), "Alice", "alice@example.com"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], @@ -349,7 +350,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Update with numeric _id") - func updateWithNumericId() { + func updateWithNumericId() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -364,7 +365,7 @@ struct MongoDBStatementGeneratorTests { originalRow: ["42", "Bob"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], @@ -377,7 +378,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Update with string _id") - func updateWithStringId() { + func updateWithStringId() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -392,7 +393,7 @@ struct MongoDBStatementGeneratorTests { originalRow: ["my-custom-id", "X"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], @@ -405,7 +406,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Update with $set and $unset") - func updateSetAndUnset() { + func updateSetAndUnset() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name", "bio"] @@ -421,7 +422,7 @@ struct MongoDBStatementGeneratorTests { originalRow: ["507f1f77bcf86cd799439011", "Alice", "Some bio"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], @@ -436,8 +437,8 @@ struct MongoDBStatementGeneratorTests { #expect(stmt.contains("\"bio\": \"\"")) } - @Test("Update skips _id column changes") - func updateSkipsIdChange() { + @Test("An edit of _id is refused, because MongoDB never changes a document's _id") + func updateRefusesIdChange() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -452,18 +453,13 @@ struct MongoDBStatementGeneratorTests { originalRow: ["old", "Alice"] ) - let results = gen.generateStatements( - from: [change], - insertedRowData: [:], - deletedRowIndices: [], - insertedRowIndices: [] - ) - - #expect(results.isEmpty) + #expect(throws: MongoDBWriteRefusal.identityChanged.refusal(ofRow: 0)) { + try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []) + } } - @Test("Update without _id in original row is skipped") - func updateNoIdSkipped() { + @Test("An update of a row with no _id is refused rather than left out") + func updateWithoutIdIsRefused() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["name", "email"] @@ -478,18 +474,13 @@ struct MongoDBStatementGeneratorTests { originalRow: ["A", "a@b.com"] ) - let results = gen.generateStatements( - from: [change], - insertedRowData: [:], - deletedRowIndices: [], - insertedRowIndices: [] - ) - - #expect(results.isEmpty) + #expect(throws: MongoDBWriteRefusal.missingIdentity.refusal(ofRow: 0)) { + try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []) + } } @Test("Update with empty cellChanges is skipped") - func updateEmptyCellChanges() { + func updateEmptyCellChanges() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -502,7 +493,7 @@ struct MongoDBStatementGeneratorTests { originalRow: ["507f1f77bcf86cd799439011", "Alice"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], @@ -515,7 +506,7 @@ struct MongoDBStatementGeneratorTests { // MARK: - DELETE @Test("Delete with ObjectId uses $oid filter") - func deleteWithObjectId() { + func deleteWithObjectId() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -529,7 +520,7 @@ struct MongoDBStatementGeneratorTests { originalRow: [.text(objectId), "Alice"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [0], @@ -543,7 +534,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Bulk delete uses deleteMany with $in") - func bulkDeleteMany() { + func bulkDeleteMany() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -557,7 +548,7 @@ struct MongoDBStatementGeneratorTests { PluginRowChange(rowIndex: 1, type: .delete, cellChanges: [], originalRow: [.text(id2), "Bob"]) ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: changes, insertedRowData: [:], deletedRowIndices: [0, 1], @@ -573,7 +564,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Bulk delete with numeric ids") - func bulkDeleteNumericIds() { + func bulkDeleteNumericIds() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -584,7 +575,7 @@ struct MongoDBStatementGeneratorTests { PluginRowChange(rowIndex: 1, type: .delete, cellChanges: [], originalRow: ["2", "Bob"]) ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: changes, insertedRowData: [:], deletedRowIndices: [0, 1], @@ -598,7 +589,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Delete quotes an _id that overflows Int64 to preserve precision") - func deleteQuotesInt64OverflowId() { + func deleteQuotesInt64OverflowId() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -611,7 +602,7 @@ struct MongoDBStatementGeneratorTests { originalRow: ["12345678901234567890", "Alice"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [0], @@ -622,7 +613,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Delete keeps a decimal or exponent _id quoted so a string _id still matches") - func deleteQuotesNonIntegerId() { + func deleteQuotesNonIntegerId() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -636,7 +627,7 @@ struct MongoDBStatementGeneratorTests { originalRow: [PluginCellValue.text(id), "Alice"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [0], @@ -649,8 +640,8 @@ struct MongoDBStatementGeneratorTests { /// An all-field filter cannot express a binary value and drops every column it cannot /// stringify, so it deletes the first partial match rather than the intended document. - @Test("A collection with no _id column produces no delete instead of an all-field match") - func singleDeleteWithoutIdColumnIsSkipped() { + @Test("A collection with no _id column refuses the delete instead of matching on every field") + func singleDeleteWithoutIdColumnIsRefused() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["name", "email"] @@ -663,18 +654,13 @@ struct MongoDBStatementGeneratorTests { originalRow: ["Alice", "alice@example.com"] ) - let results = gen.generateStatements( - from: [change], - insertedRowData: [:], - deletedRowIndices: [0], - insertedRowIndices: [] - ) - - #expect(results.isEmpty) + #expect(throws: MongoDBWriteRefusal.missingIdentity.refusal(ofRow: 0)) { + try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: []) + } } @Test("Delete not in deletedRowIndices is skipped") - func deleteNotInIndicesSkipped() { + func deleteNotInIndicesSkipped() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -687,7 +673,7 @@ struct MongoDBStatementGeneratorTests { originalRow: ["507f1f77bcf86cd799439011", "Alice"] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [0], // does not contain 5 @@ -697,8 +683,8 @@ struct MongoDBStatementGeneratorTests { #expect(results.isEmpty) } - @Test("Delete without originalRow is skipped") - func deleteNoOriginalRowSkipped() { + @Test("A delete without its original row is refused") + func deleteNoOriginalRowIsRefused() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name"] @@ -711,20 +697,15 @@ struct MongoDBStatementGeneratorTests { originalRow: nil ) - let results = gen.generateStatements( - from: [change], - insertedRowData: [:], - deletedRowIndices: [0], - insertedRowIndices: [] - ) - - #expect(results.isEmpty) + #expect(throws: MongoDBWriteRefusal.missingIdentity.refusal(ofRow: 0)) { + try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: []) + } } // MARK: - Mixed Operations @Test("Mixed insert, update, and delete in one batch") - func mixedOperations() { + func mixedOperations() throws { let gen = MongoDBStatementGenerator( collectionName: "users", columns: ["_id", "name", "email"] @@ -758,7 +739,7 @@ struct MongoDBStatementGeneratorTests { 0: [nil, "Alice", "alice@test.com"] ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: changes, insertedRowData: insertedData, deletedRowIndices: [2], @@ -774,7 +755,7 @@ struct MongoDBStatementGeneratorTests { // MARK: - Collection Accessor @Test("Collection with dots goes through getCollection") - func collectionBracketNotation() { + func collectionBracketNotation() throws { let gen = MongoDBStatementGenerator( collectionName: "my.collection", columns: ["_id", "name"] @@ -787,7 +768,7 @@ struct MongoDBStatementGeneratorTests { originalRow: nil ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [0: [nil, "Test"]], deletedRowIndices: [], @@ -801,7 +782,7 @@ struct MongoDBStatementGeneratorTests { // MARK: - Value Type Detection @Test("Boolean values are serialized as booleans") - func booleanSerialization() { + func booleanSerialization() throws { let gen = MongoDBStatementGenerator( collectionName: "data", columns: ["_id", "active"] @@ -814,7 +795,7 @@ struct MongoDBStatementGeneratorTests { originalRow: nil ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [0: [nil, "true"]], deletedRowIndices: [], @@ -826,7 +807,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Float values are serialized as numbers") - func floatSerialization() { + func floatSerialization() throws { let gen = MongoDBStatementGenerator( collectionName: "data", columns: ["_id", "price"] @@ -839,7 +820,7 @@ struct MongoDBStatementGeneratorTests { originalRow: nil ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [0: [nil, "19.99"]], deletedRowIndices: [], @@ -850,8 +831,8 @@ struct MongoDBStatementGeneratorTests { #expect(results[0].statement.contains("\"price\": 19.99")) } - @Test("JSON object values are passed through as-is") - func jsonObjectPassthrough() { + @Test("A JSON object value is written as the object it spells") + func jsonObjectPassthrough() throws { let gen = MongoDBStatementGenerator( collectionName: "data", columns: ["_id", "metadata"] @@ -864,7 +845,7 @@ struct MongoDBStatementGeneratorTests { originalRow: nil ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [0: [nil, "{\"nested\": true}"]], deletedRowIndices: [], @@ -872,11 +853,11 @@ struct MongoDBStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement.contains("\"metadata\": {\"nested\": true}")) + #expect(results[0].statement.contains("\"metadata\": {\"nested\":true}")) } - @Test("JSON array values are passed through as-is") - func jsonArrayPassthrough() { + @Test("A JSON array value is written as the array it spells") + func jsonArrayPassthrough() throws { let gen = MongoDBStatementGenerator( collectionName: "data", columns: ["_id", "tags"] @@ -889,7 +870,7 @@ struct MongoDBStatementGeneratorTests { originalRow: nil ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [0: [nil, "[1, 2, 3]"]], deletedRowIndices: [], @@ -897,7 +878,7 @@ struct MongoDBStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement.contains("\"tags\": [1, 2, 3]")) + #expect(results[0].statement.contains("\"tags\": [1,2,3]")) } // MARK: - Binary UUID round trip @@ -906,7 +887,7 @@ struct MongoDBStatementGeneratorTests { private static let standardBase64 = "jNAD60olQySTMoj84toNGg==" @Test("Editing a legacy UUID field writes BSON binary, not a string") - func updateWritesLegacyUuidBinary() { + func updateWritesLegacyUuidBinary() throws { let gen = MongoDBStatementGenerator(collectionName: "docs", columns: ["_id", "ref"]) let change = PluginRowChange( rowIndex: 0, @@ -922,7 +903,7 @@ struct MongoDBStatementGeneratorTests { originalRow: ["507f1f77bcf86cd799439011", .null] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] ) @@ -934,11 +915,11 @@ struct MongoDBStatementGeneratorTests { } @Test("Inserting a standard UUID writes BSON binary subtype 4") - func insertWritesStandardUuidBinary() { + func insertWritesStandardUuidBinary() throws { let gen = MongoDBStatementGenerator(collectionName: "docs", columns: ["_id", "ref"]) let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [0: [nil, .text("UUID(\"\(Self.uuid)\")")]], deletedRowIndices: [], @@ -951,7 +932,7 @@ struct MongoDBStatementGeneratorTests { } @Test("An _id that is a legacy UUID filters on binary, not on the wrapper text") - func updateFiltersOnBinaryId() { + func updateFiltersOnBinaryId() throws { let gen = MongoDBStatementGenerator(collectionName: "docs", columns: ["_id", "name"]) let change = PluginRowChange( rowIndex: 0, @@ -962,7 +943,7 @@ struct MongoDBStatementGeneratorTests { originalRow: [.text("LegacyJavaUUID(\"\(Self.uuid)\")"), .text("a")] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] ) @@ -973,7 +954,7 @@ struct MongoDBStatementGeneratorTests { } @Test("Deleting a document with a legacy UUID _id filters on binary") - func deleteFiltersOnBinaryId() { + func deleteFiltersOnBinaryId() throws { let gen = MongoDBStatementGenerator(collectionName: "docs", columns: ["_id", "name"]) let change = PluginRowChange( rowIndex: 0, @@ -982,7 +963,7 @@ struct MongoDBStatementGeneratorTests { originalRow: [.text("LegacyJavaUUID(\"\(Self.uuid)\")"), .text("a")] ) - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: [] ) @@ -991,7 +972,7 @@ struct MongoDBStatementGeneratorTests { } @Test("A bulk delete of UUID _ids uses binary values inside $in") - func bulkDeleteUsesBinaryIds() { + func bulkDeleteUsesBinaryIds() throws { let gen = MongoDBStatementGenerator(collectionName: "docs", columns: ["_id"]) let changes = [ PluginRowChange( @@ -1004,7 +985,7 @@ struct MongoDBStatementGeneratorTests { ) ] - let results = gen.generateStatements( + let results = try gen.generateRowWrites( from: changes, insertedRowData: [:], deletedRowIndices: [0, 1], insertedRowIndices: [] ) @@ -1017,8 +998,8 @@ struct MongoDBStatementGeneratorTests { /// Matching on the remaining fields cannot express a binary value, so it would /// delete the first partial match instead of the intended document. - @Test("A delete without a usable _id is skipped rather than matching on other fields") - func deleteWithoutIdIsSkipped() { + @Test("A delete whose binary _id has no known subtype is refused rather than matched on other fields") + func deleteWithoutUsableIdIsRefused() throws { let gen = MongoDBStatementGenerator(collectionName: "docs", columns: ["_id", "name"]) let change = PluginRowChange( rowIndex: 0, @@ -1027,11 +1008,9 @@ struct MongoDBStatementGeneratorTests { originalRow: [.bytes(Data([0x01, 0x02])), .text("Alice")] ) - let results = gen.generateStatements( - from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: [] - ) - - #expect(results.isEmpty) + #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "_id").refusal(ofRow: 0)) { + try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: []) + } } // MARK: - Restore @@ -1040,7 +1019,7 @@ struct MongoDBStatementGeneratorTests { /// requirement: a new `_id` is a different document, and whatever referenced the old one is /// still pointing at nothing. @Test("Restoring a deleted document keeps its original _id") - func restoreKeepsObjectId() { + func restoreKeepsObjectId() throws { let gen = MongoDBStatementGenerator(collectionName: "users", columns: ["_id", "name"]) let statements = gen.generateRestore(rows: [["507f1f77bcf86cd799439011", "Alice"]]) @@ -1054,7 +1033,7 @@ struct MongoDBStatementGeneratorTests { } @Test("A numeric key is restored as a number, not a string") - func restoreKeepsNumericId() { + func restoreKeepsNumericId() throws { let gen = MongoDBStatementGenerator(collectionName: "counters", columns: ["_id", "value"]) let statement = gen.generateRestore(rows: [["42", "7"]])?.first?.statement ?? "" @@ -1063,21 +1042,33 @@ struct MongoDBStatementGeneratorTests { #expect(document?["_id"] as? Int == 42) } + @Test("A restored document keeps its binary field with the subtype it was read with") + func restoreKeepsBinaryField() throws { + var subtypes = MongoDBBinarySubtypes.empty + subtypes.record(Data([0x01]), subtype: 5, field: "avatar") + let gen = MongoDBStatementGenerator( + collectionName: "users", columns: ["_id", "avatar"], binarySubtypes: subtypes + ) + + let statement = try #require(gen.generateRestore(rows: [["507f1f77bcf86cd799439011", .bytes(Data([0x01]))]])?.first) + + #expect(statement.statement.contains(#""avatar": {"$binary": {"base64": "AQ==", "subType": "05"}}"#)) + } + /// Dropping the field would restore a document that is missing it, and report success. - @Test("A document with a binary field is refused rather than restored without it") - func restoreRefusesBinaryField() { + @Test("A binary field whose subtype is unknown refuses the restore rather than dropping the field") + func restoreRefusesUnknownBinarySubtype() throws { let gen = MongoDBStatementGenerator(collectionName: "users", columns: ["_id", "avatar"]) #expect(gen.generateRestore(rows: [["507f1f77bcf86cd799439011", .bytes(Data([0x01]))]]) == nil) } @Test("A collection with no _id column cannot be restored") - func restoreRefusesWithoutIdColumn() { + func restoreRefusesWithoutIdColumn() throws { let gen = MongoDBStatementGenerator(collectionName: "users", columns: ["name", "email"]) #expect(gen.generateRestore(rows: [["Alice", "alice@example.com"]]) == nil) } - } private func firstArgumentObject(in statement: String) -> [String: Any]? { diff --git a/TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift b/TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift new file mode 100644 index 0000000000..f439b22ba1 --- /dev/null +++ b/TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift @@ -0,0 +1,233 @@ +// +// MongoDBWriteBackBinaryTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct MongoDBWriteBackBinaryTests { + private static let objectId = "507f1f77bcf86cd799439011" + private static let png = Data([0x89, 0x50]) + private static let signature = Data([0x00, 0x01, 0x02, 0x03]) + + private func generator( + columns: [String] = ["_id", "name", "thumbnail"], + subtypes: MongoDBBinarySubtypes = .empty, + declaredBinary: Set = [], + identityKind: BsonValueKind? = nil + ) -> MongoDBStatementGenerator { + MongoDBStatementGenerator( + collectionName: "items", + columns: columns, + identityKind: identityKind, + binarySubtypes: subtypes, + declaredBinaryFields: declaredBinary + ) + } + + private func subtypes(_ entries: [(Data, UInt8, String)]) -> MongoDBBinarySubtypes { + var recorded = MongoDBBinarySubtypes.empty + for (data, subtype, field) in entries { + recorded.record(data, subtype: subtype, field: field) + } + return recorded + } + + private func insert(_ values: [PluginCellValue], with gen: MongoDBStatementGenerator) throws -> [PluginRowWrite] { + try gen.generateRowWrites( + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + insertedRowData: [0: values], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + } + + private func edit( + _ column: String, + from oldValue: PluginCellValue, + to newValue: PluginCellValue, + with gen: MongoDBStatementGenerator + ) throws -> [PluginRowWrite] { + let change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: 2, columnName: column, oldValue: oldValue, newValue: newValue)], + originalRow: [.text(Self.objectId), .text("a"), oldValue] + ) + return try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []) + } + + // MARK: - Inserts + + @Test("A duplicated row keeps its binary field, with the subtype the value was read with") + func duplicateKeepsGenericBinary() throws { + let gen = generator(subtypes: subtypes([(Self.png, 0, "thumbnail")])) + + let writes = try insert(["__DEFAULT__", "a", .bytes(Self.png)], with: gen) + + #expect(writes.count == 1) + #expect(writes[0].statement.contains(#""thumbnail": {"$binary": {"base64": "iVA=", "subType": "00"}}"#)) + #expect(writes[0].rowIndices == [0]) + } + + @Test("A subtype-5 value is written back as subtype 05") + func duplicateKeepsSubtypeFive() throws { + let gen = generator(columns: ["_id", "name", "sig"], subtypes: subtypes([(Self.signature, 5, "sig")])) + + let writes = try insert(["__DEFAULT__", "b", .bytes(Self.signature)], with: gen) + + #expect(writes[0].statement.contains(#""sig": {"$binary": {"base64": "AAECAw==", "subType": "05"}}"#)) + } + + @Test("Bytes seen with two subtypes in one field are refused, never inserted without them") + func ambiguousSubtypeIsRefused() { + let gen = generator(subtypes: subtypes([(Self.png, 0, "thumbnail"), (Self.png, 5, "thumbnail")])) + + #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "thumbnail").refusal(ofRow: 0)) { + try insert(["__DEFAULT__", "a", .bytes(Self.png)], with: gen) + } + } + + @Test("Bytes never read and in a field no validator declares binary are refused") + func unknownBytesAreRefused() { + #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "thumbnail").refusal(ofRow: 0)) { + try insert(["__DEFAULT__", "a", .bytes(Self.png)], with: generator()) + } + } + + @Test("New bytes in a field the validator declares binary are generic binary") + func declaredBinaryFieldTakesSubtypeZero() throws { + let gen = generator(declaredBinary: ["thumbnail"]) + + let writes = try insert(["__DEFAULT__", "a", .bytes(Self.png)], with: gen) + + #expect(writes[0].statement.contains(#""subType": "00""#)) + } + + @Test("A row empty apart from a value that cannot be written is refused, not inserted as {}") + func refusedValueNeverBecomesEmptyDocument() { + #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "thumbnail").refusal(ofRow: 0)) { + try insert([nil, nil, .bytes(Self.png)], with: generator()) + } + } + + @Test("An _id typed into a new row is kept, typed the way the filters type it") + func typedIdIsKept() throws { + let writes = try insert(["1001", "a", nil], with: generator(identityKind: .int32)) + + #expect(writes[0].statement == #"db.items.insertOne({"_id": 1001, "name": "a"})"#) + } + + // MARK: - Updates + + @Test("Editing a binary cell writes $set with the bytes, never $unset") + func editedBinaryIsSet() throws { + let gen = generator(subtypes: subtypes([(Self.png, 0, "thumbnail")])) + + let writes = try edit("thumbnail", from: .bytes(Self.png), to: .bytes(Data([0x01, 0x02])), with: gen) + + #expect(writes.count == 1) + let statement = writes[0].statement + #expect(statement.contains(#""$set": {"thumbnail": {"$binary": {"base64": "AQI=", "subType": "00"}}}"#)) + #expect(!statement.contains("$unset")) + } + + @Test("Edited bytes keep the subtype of the value they replace, not one recorded for other bytes") + func editKeepsTheReplacedValuesSubtype() throws { + let gen = generator(subtypes: subtypes([(Self.png, 5, "thumbnail"), (Data([0x01]), 0, "thumbnail")])) + + let writes = try edit("thumbnail", from: .bytes(Self.png), to: .bytes(Data([0x01])), with: gen) + + #expect(writes[0].statement.contains(#""subType": "05""#)) + } + + @Test("Emptying a binary cell writes empty bytes of the same subtype") + func emptyTextOverBinaryIsEmptyBinary() throws { + let gen = generator(subtypes: subtypes([(Self.png, 5, "thumbnail")])) + + let writes = try edit("thumbnail", from: .bytes(Self.png), to: .text(""), with: gen) + + #expect(writes[0].statement.contains(#""thumbnail": {"$binary": {"base64": "", "subType": "05"}}"#)) + } + + @Test("Text typed over a binary value is refused rather than stored as a string") + func textOverBinaryIsRefused() { + let gen = generator(subtypes: subtypes([(Self.png, 0, "thumbnail")])) + + #expect(throws: MongoDBWriteRefusal.binaryNeedsBytes(field: "thumbnail").refusal(ofRow: 0)) { + try edit("thumbnail", from: .bytes(Self.png), to: .text("hello"), with: gen) + } + } + + @Test("DEFAULT is refused on an update, because MongoDB has no default values") + func defaultMarkerIsRefused() { + #expect(throws: MongoDBWriteRefusal.noDefaultValue(field: "thumbnail").refusal(ofRow: 0)) { + try edit("thumbnail", from: .text("x"), to: .text("__DEFAULT__"), with: generator()) + } + } + + // MARK: - Binary _id + + @Test("A document keyed by generic binary is updated and deleted through a $binary filter") + func binaryIdFiltersOnBinary() throws { + let key = Data([0xAB, 0xCD, 0xEF]) + let gen = generator(columns: ["_id", "name"], subtypes: subtypes([(key, 5, "_id")])) + let update = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: 1, columnName: "name", oldValue: .text("k"), newValue: .text("k2"))], + originalRow: [.bytes(key), .text("k")] + ) + let delete = PluginRowChange(rowIndex: 1, type: .delete, cellChanges: [], originalRow: [.bytes(key), .text("k")]) + + let writes = try gen.generateRowWrites( + from: [update, delete], insertedRowData: [:], deletedRowIndices: [1], insertedRowIndices: [] + ) + + let filter = #"{"_id": {"$binary": {"base64": "q83v", "subType": "05"}}}"# + #expect(writes.map { $0.statement } == [ + "db.items.updateOne(\(filter), {\"$set\": {\"name\": \"k2\"}})", + "db.items.deleteOne(\(filter))" + ]) + #expect(writes.map { $0.rowIndices } == [[0], [1]]) + } + + @Test("A binary _id with no recorded subtype uses the subtype every sampled _id shares") + func binaryIdFallsBackToTheSharedSubtype() throws { + let gen = generator(columns: ["_id", "name"], identityKind: .binary(subtype: 3)) + let delete = PluginRowChange(rowIndex: 0, type: .delete, cellChanges: [], originalRow: [.bytes(Data([1])), "k"]) + + let writes = try gen.generateRowWrites(from: [delete], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: []) + + #expect(writes.first?.statement.contains(#""subType": "03""#) == true) + } + + // MARK: - Recording + + @Test("Every top-level binary value is recorded by field and value") + func recordingReadsTopLevelBinary() { + let recorded = MongoDBBinarySubtypes.recording([ + ["_id": MongoDBObjectId(hex: Self.objectId), "sig": MongoDBBinaryValue(data: Self.signature, subtype: 5)], + ["thumb": MongoDBBinaryValue(data: Self.png, subtype: 0), "nested": ["sig": MongoDBBinaryValue(data: Self.png, subtype: 9)]] + ]) + + #expect(recorded.subtypes(of: Self.signature, in: "sig") == [5]) + #expect(recorded.subtypes(of: Self.png, in: "thumb") == [0]) + #expect(recorded.subtypes(of: Self.png, in: "sig").isEmpty) + #expect(recorded.count == 2) + } + + @Test("Merging a later page keeps earlier values and keeps a value seen twice ambiguous") + func mergingAddsRatherThanReplaces() { + let first = subtypes([(Self.png, 0, "blob")]) + let second = subtypes([(Self.png, 5, "blob"), (Self.signature, 5, "blob")]) + + let merged = first.merging(second) + + #expect(merged.subtypes(of: Self.png, in: "blob") == [0, 5]) + #expect(merged.subtypes(of: Self.signature, in: "blob") == [5]) + #expect(merged.count == 3) + } +} diff --git a/TableProTests/Plugins/MongoDBWriteBackTypeTests.swift b/TableProTests/Plugins/MongoDBWriteBackTypeTests.swift index 9d9781817d..44a6e8e30d 100644 --- a/TableProTests/Plugins/MongoDBWriteBackTypeTests.swift +++ b/TableProTests/Plugins/MongoDBWriteBackTypeTests.swift @@ -13,7 +13,7 @@ struct MongoDBWriteBackTypeTests { to newValue: PluginCellValue, kinds: [String: BsonValueKind], declared: [String: BsonValueKind] = [:] - ) -> String? { + ) throws -> String? { let gen = MongoDBStatementGenerator( collectionName: "products", columns: ["_id", column], @@ -26,14 +26,14 @@ struct MongoDBWriteBackTypeTests { cellChanges: [(columnIndex: 1, columnName: column, oldValue: .null, newValue: newValue)], originalRow: [.text("507f1f77bcf86cd799439011"), .null] ) - return gen.generateStatements( + return try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] ).first?.statement } @Test("A decimal column stays decimal instead of collapsing to a double") func decimalColumnKeepsItsType() throws { - let stmt = try #require(update( + let stmt = try #require(try update( column: "price", to: .text("1847.270000000000000000000000001"), kinds: ["price": .decimal128] @@ -43,44 +43,46 @@ struct MongoDBWriteBackTypeTests { @Test("A 64-bit integer column is not narrowed to int32") func int64ColumnKeepsItsWidth() throws { - let stmt = try #require(update(column: "qty", to: .text("5"), kinds: ["qty": .int64])) + let stmt = try #require(try update(column: "qty", to: .text("5"), kinds: ["qty": .int64])) #expect(stmt.contains(#"{"$numberLong": "5"}"#)) } @Test("A double column stays a double even when the value looks whole") func doubleColumnKeepsItsType() throws { - let stmt = try #require(update(column: "rate", to: .text("3"), kinds: ["rate": .double])) + let stmt = try #require(try update(column: "rate", to: .text("3"), kinds: ["rate": .double])) #expect(stmt.contains(#"{"$numberDouble": "3"}"#)) } @Test("A column with no recorded type writes a bare number, as before") func unknownColumnWritesBareNumber() throws { - let stmt = try #require(update(column: "count", to: .text("7"), kinds: [:])) + let stmt = try #require(try update(column: "count", to: .text("7"), kinds: [:])) #expect(stmt.contains(#""count": 7"#)) } @Test("A truncated nested value is refused instead of stored as a fragment") func truncatedValueIsNotWritten() { let fragment = "{\"a\": 1, \"b\": \"" + String(repeating: "x", count: 50) + "..." - #expect(update(column: "payload", to: .text(fragment), kinds: [:]) == nil) + #expect(throws: MongoDBWriteRefusal.truncatedValue(field: "payload").refusal(ofRow: 0)) { + try update(column: "payload", to: .text(fragment), kinds: [:]) + } } @Test("A complete nested value still writes") func completeNestedValueWrites() throws { - let stmt = try #require(update(column: "payload", to: .text(#"{"a":1}"#), kinds: [:])) + let stmt = try #require(try update(column: "payload", to: .text(#"{"a":1}"#), kinds: [:])) #expect(stmt.contains(#""payload": {"a":1}"#)) } @Test("A decimal wider than a 64-bit integer still writes as a decimal") func wideDecimalKeepsItsType() throws { let digits = "12345678901234567890123456789012" - let stmt = try #require(update(column: "price", to: .text(digits), kinds: ["price": .decimal128])) + let stmt = try #require(try update(column: "price", to: .text(digits), kinds: ["price": .decimal128])) #expect(stmt.contains("{\"$numberDecimal\": \"\(digits)\"}")) } @Test("A half-typed brace is written, not mistaken for truncated text") func userTypedBraceStillWrites() throws { - let stmt = try #require(update(column: "note", to: .text("{draft"), kinds: [:])) + let stmt = try #require(try update(column: "note", to: .text("{draft"), kinds: [:])) #expect(stmt.contains(#""note": "{draft""#)) } @@ -88,13 +90,13 @@ struct MongoDBWriteBackTypeTests { @Test("A date column is written as a BSON date, not the text the grid shows") func dateColumnWritesADate() throws { - let stmt = try #require(update(column: "when", to: .text("2024-01-02T03:04:05Z"), kinds: ["when": .date])) + let stmt = try #require(try update(column: "when", to: .text("2024-01-02T03:04:05Z"), kinds: ["when": .date])) #expect(stmt.contains(#""when": {"$date": {"$numberLong": "1704164645000"}}"#)) } @Test("A date the picker wrote is read in the local zone, which is the zone the picker shows") func pickedDateIsLocalWallClock() throws { - let stmt = try #require(update(column: "when", to: .text("2024-01-02 03:04:05"), kinds: ["when": .date])) + let stmt = try #require(try update(column: "when", to: .text("2024-01-02 03:04:05"), kinds: ["when": .date])) let formatter = DateFormatter() formatter.locale = Locale(identifier: "en_US_POSIX") formatter.timeZone = .current @@ -106,19 +108,19 @@ struct MongoDBWriteBackTypeTests { @Test("A date-only value is a date too") func dateOnlyIsADate() throws { - let stmt = try #require(update(column: "day", to: .text("2024-01-02"), kinds: ["day": .date])) + let stmt = try #require(try update(column: "day", to: .text("2024-01-02"), kinds: ["day": .date])) #expect(stmt.contains(#""day": {"$date": "#)) } @Test("A bare number typed into a date column stays a number rather than becoming 1970") func numberInADateColumnStaysANumber() throws { - let stmt = try #require(update(column: "day", to: .text("2024"), kinds: ["day": .date])) + let stmt = try #require(try update(column: "day", to: .text("2024"), kinds: ["day": .date])) #expect(stmt.contains(#""day": 2024"#)) } @Test("A field holding ObjectIds keeps holding ObjectIds") func objectIdColumnWritesAnObjectId() throws { - let stmt = try #require(update( + let stmt = try #require(try update( column: "authorId", to: .text("65a1b2c3d4e5f60718293a4b"), kinds: ["authorId": .objectId] )) #expect(stmt.contains(#""authorId": {"$oid": "65a1b2c3d4e5f60718293a4b"}"#)) @@ -129,7 +131,7 @@ struct MongoDBWriteBackTypeTests { @Test("A field the validator declares a string is written as one, whatever the text looks like") func declaredStringIsAlwaysAString() throws { for text in ["123", "true", "null", "[1, 2]"] { - let stmt = try #require(update(column: "code", to: .text(text), kinds: [:], declared: ["code": .string])) + let stmt = try #require(try update(column: "code", to: .text(text), kinds: [:], declared: ["code": .string])) let escaped = text.replacingOccurrences(of: "\"", with: "\\\"") #expect(stmt.contains("\"code\": \"\(escaped)\""), "\(text)") } @@ -137,13 +139,13 @@ struct MongoDBWriteBackTypeTests { @Test("A sampled string column still reads typed JSON, because sampling is a majority vote") func sampledStringKeepsItsSpelling() throws { - let stmt = try #require(update(column: "meta", to: .text(#"{"a":1}"#), kinds: ["meta": .string])) + let stmt = try #require(try update(column: "meta", to: .text(#"{"a":1}"#), kinds: ["meta": .string])) #expect(stmt.contains(#""meta": {"a":1}"#)) } @Test("A declared type outranks what the sampled documents held") func declaredKindWins() throws { - let stmt = try #require(update( + let stmt = try #require(try update( column: "rate", to: .text("3"), kinds: ["rate": .int32], declared: ["rate": .double] )) #expect(stmt.contains(#"{"$numberDouble": "3"}"#)) @@ -151,7 +153,7 @@ struct MongoDBWriteBackTypeTests { @Test("An int field takes a bare number that fits in 32 bits") func int32ColumnWritesABareNumber() throws { - let stmt = try #require(update(column: "n", to: .text("42"), kinds: [:], declared: ["n": .int32])) + let stmt = try #require(try update(column: "n", to: .text("42"), kinds: [:], declared: ["n": .int32])) #expect(stmt.contains(#""n": 42"#)) } @@ -161,13 +163,13 @@ struct MongoDBWriteBackTypeTests { /// `9007199254740992`, so the bare literal reached the server one off. @Test("An integer past 2^53 is written as a 64-bit integer so JavaScript cannot round it") func hugeIntegerIsNotRounded() throws { - let stmt = try #require(update(column: "big", to: .text("9007199254740993"), kinds: [:])) + let stmt = try #require(try update(column: "big", to: .text("9007199254740993"), kinds: [:])) #expect(stmt.contains(#""big": {"$numberLong": "9007199254740993"}"#)) } @Test("An integer JavaScript holds exactly stays bare") func exactIntegerStaysBare() throws { - let stmt = try #require(update(column: "n", to: .text("9007199254740992"), kinds: [:])) + let stmt = try #require(try update(column: "n", to: .text("9007199254740992"), kinds: [:])) #expect(stmt.contains(#""n": 9007199254740992"#)) } @@ -178,7 +180,7 @@ struct MongoDBWriteBackTypeTests { @Test("Stored text shaped like an array but not JSON is written as a string, never run") func arrayShapedCodeIsAString() throws { let hostile = #"[db.getCollection("audit").drop()]"# - let stmt = try #require(update(column: "note", to: .text(hostile), kinds: [:])) + let stmt = try #require(try update(column: "note", to: .text(hostile), kinds: [:])) #expect(stmt.contains("\"note\": \(MongoScriptJson.jsonString(hostile))")) #expect(!stmt.contains(#""note": [db"#)) } @@ -186,7 +188,7 @@ struct MongoDBWriteBackTypeTests { @Test("Stored text shaped like a document but not JSON is written as a string, never run") func documentShapedCodeIsAString() throws { let hostile = #"{a: db.dropDatabase()}"# - let stmt = try #require(update(column: "note", to: .text(hostile), kinds: [:])) + let stmt = try #require(try update(column: "note", to: .text(hostile), kinds: [:])) #expect(stmt.contains("\"note\": \(MongoScriptJson.jsonString(hostile))")) } @@ -195,7 +197,7 @@ struct MongoDBWriteBackTypeTests { @Test("A quote joined to a Prepend character is still escaped, so the value cannot close its string") func quoteInsideAGraphemeClusterIsEscaped() throws { let hostile = "x\u{0600}\"}}); db.getCollection(\"victim\").drop(); db.x.find({\"a\": {\"b\": \"" - let stmt = try #require(update(column: "note", to: .text(hostile), kinds: [:], declared: ["note": .string])) + let stmt = try #require(try update(column: "note", to: .text(hostile), kinds: [:], declared: ["note": .string])) let start = try #require(stmt.range(of: "\"$set\": ")).upperBound let setDocument = String(stmt[start...].dropLast(2)) let data = try #require(setDocument.data(using: .utf8)) @@ -214,7 +216,7 @@ struct MongoDBWriteBackTypeTests { "[1], db.dropDatabase(), [2]" ] for hostile in payloads { - let stmt = try #require(update(column: "note", to: .text(hostile), kinds: [:])) + let stmt = try #require(try update(column: "note", to: .text(hostile), kinds: [:])) #expect(stmt.contains("\"note\": \(MongoScriptJson.jsonString(hostile))"), "\(hostile)") let gen = MongoDBStatementGenerator(collectionName: "notes", columns: ["_id", "note"]) let restored = try #require(gen.generateRestore(rows: [["507f1f77bcf86cd799439011", .text(hostile)]])?.first) @@ -236,7 +238,7 @@ struct MongoDBWriteBackTypeTests { id: String, idKind: BsonValueKind?, sampledKinds: [String: BsonValueKind] = [:] - ) -> String? { + ) throws -> String? { let gen = MongoDBStatementGenerator( collectionName: "codes", columns: ["_id", "label"], @@ -249,32 +251,32 @@ struct MongoDBWriteBackTypeTests { cellChanges: [(columnIndex: 1, columnName: "label", oldValue: .text("a"), newValue: .text("b"))], originalRow: [.text(id), .text("a")] ) - return gen.generateStatements( + return try gen.generateRowWrites( from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] ).first?.statement } @Test("A string _id that looks like a number is matched as a string") func stringIdIsMatchedAsAString() throws { - let stmt = try #require(updateFilter(id: "1001", idKind: .string)) + let stmt = try #require(try updateFilter(id: "1001", idKind: .string)) #expect(stmt.contains(#"{"_id": "1001"}"#)) } @Test("A string _id that looks like an ObjectId is matched as a string") func hexStringIdIsMatchedAsAString() throws { - let stmt = try #require(updateFilter(id: "65a1b2c3d4e5f60718293a4b", idKind: .string)) + let stmt = try #require(try updateFilter(id: "65a1b2c3d4e5f60718293a4b", idKind: .string)) #expect(stmt.contains(#"{"_id": "65a1b2c3d4e5f60718293a4b"}"#)) } @Test("A 64-bit _id is matched as one") func longIdIsMatchedAsALong() throws { - let stmt = try #require(updateFilter(id: "42", idKind: .int64)) + let stmt = try #require(try updateFilter(id: "42", idKind: .int64)) #expect(stmt.contains(#"{"_id": {"$numberLong": "42"}}"#)) } @Test("A mostly-string _id column does not quote a row whose _id kinds differ") func mixedIdKindsFallBackToTheSpelling() throws { - let stmt = try #require(updateFilter( + let stmt = try #require(try updateFilter( id: "65a1b2c3d4e5f60718293a4b", idKind: nil, sampledKinds: ["_id": .string] )) #expect(stmt.contains(#"{"_id": {"$oid": "65a1b2c3d4e5f60718293a4b"}}"#)) @@ -283,15 +285,15 @@ struct MongoDBWriteBackTypeTests { /// JavaScriptCore evaluates a bare `010` in a statement as the octal number 8. @Test("An int written with a leading zero is sent as its decimal value") func int32IsCanonicalised() throws { - let stmt = try #require(update(column: "n", to: .text("010"), kinds: [:], declared: ["n": .int32])) + let stmt = try #require(try update(column: "n", to: .text("010"), kinds: [:], declared: ["n": .int32])) #expect(stmt.contains(#""n": 10"#)) - let negative = try #require(update(column: "n", to: .text("-010"), kinds: ["n": .int32])) + let negative = try #require(try update(column: "n", to: .text("-010"), kinds: ["n": .int32])) #expect(negative.contains(#""n": -10"#)) } @Test("An _id of unknown kind is still read from its spelling") func unknownIdFallsBackToItsSpelling() throws { - let stmt = try #require(updateFilter(id: "65a1b2c3d4e5f60718293a4b", idKind: nil)) + let stmt = try #require(try updateFilter(id: "65a1b2c3d4e5f60718293a4b", idKind: nil)) #expect(stmt.contains(#"{"_id": {"$oid": "65a1b2c3d4e5f60718293a4b"}}"#)) } } diff --git a/docs/databases/mongodb.mdx b/docs/databases/mongodb.mdx index 8145037bd8..8456a1955c 100644 --- a/docs/databases/mongodb.mdx +++ b/docs/databases/mongodb.mdx @@ -73,10 +73,14 @@ Over an SSH tunnel only the first host is used and the rest of the list is dropp ## Browsing collections -Click a collection to page through its documents. The columns are the fields in the documents on screen, then any field the collection's `$jsonSchema` validator declares that none of them hold, so an empty collection still shows the fields it was created with. ObjectIds render as strings, arrays and nested objects as formatted JSON. +Click a collection to page through its documents. The columns are the fields in the documents on screen, then any field the collection's `$jsonSchema` validator declares that none of them hold, so an empty collection still shows the fields it was created with. A top-level ObjectId renders as its hex string. A nested object or array shows as Extended JSON in the order it is stored: `{"$oid": "…"}`, `{"$date": "2024-05-01T10:00:00.123Z"}`, `{"$numberLong": "5"}`, and `3.0` for a double that holds a whole number. An edited cell keeps its field's type. A date stays a date, whether typed as `2024-01-02T03:04:05Z`, as `2024-01-02`, or picked in the date picker, which writes your local time. An ObjectId stays an ObjectId, an integer past 2^53 is sent as a 64-bit integer, and a field the validator declares as `string` is written as text even when it reads `123` or `true`. Where the grid has no type to go on, `123` is a number and `{"a": 1}` a document; text that only looks like JSON is written as a string. +Editing a nested object or array writes only the paths that changed, such as `address.city` or `tags.1`, so every other value in it keeps its type and its place. Adding two keys at once, reordering keys, or changing an array's length writes the whole value instead. A binary cell keeps its BSON subtype when its bytes are edited, and a duplicated or pasted row keeps its binary fields. **Set NULL** removes the field from the document, and a new row saved with every cell empty inserts a document holding only a generated `_id`. + +A field whose own name contains a dot, starts with `$` or is `__proto__` is written with `$setField`, which needs MongoDB 5.0 or later. On an older server the save is refused and nothing is sent. + The filter bar's column picker lists paths inside nested objects and arrays of objects, so `customer.country` and `items.sku` filter directly; a row on an array field chooses **any element** or **same element**, which makes one array entry satisfy every row set to it. See [Filtering](/features/filtering#nested-fields). A field name containing a literal dot is left out of the picker, since MongoDB reads a dot as a path separator; reach it with `$getField` inside `$expr`. The Structure tab lists a collection's indexes; drop one from a query tab with `db.users.dropIndex("email_1")`. **New Database** asks for a database name and a first collection, both required. **New View** opens a query tab holding a `db.createView("view_name", "source_collection", [pipeline])` template, and editing a view pre-fills `db.runCommand({"collMod": …})`. @@ -209,7 +213,8 @@ New connections default to **Disabled**, and the driver has no TLS fallback: **P ## Limitations -- A row with no `_id` cannot be updated or deleted. The save is skipped rather than matched on the remaining fields. Keep `_id` in the projection so every row carries one. +- A row with no `_id` cannot be updated or deleted. The save is refused rather than matched on the remaining fields, and every change stays pending. Keep `_id` in the projection so every row carries one. +- A binary value keeps the subtype it was read with. New bytes in a field whose validator does not declare `binData`, or the same bytes read with two different subtypes, refuse the save: write that value with a query. - `_id` is read-only in the grid, and a row added with **Add Row** is inserted without one so the server generates it. To choose your own, use **Insert Document…**. - Transactions are not exposed. Statements always run standalone, on any topology. - A collection takes one text index. A second **FULLTEXT** row fails after the collection and the indexes before it are created: list every text field in one index instead. @@ -218,7 +223,7 @@ New connections default to **Disabled**, and the driver has no TLS fallback: **P - **same element** covers a field one array deep. A path through an array inside another array needs nested `$elemMatch`, so those filter with dot notation only. - GridFS buckets are not browsable, and change streams are unsupported. - A script that loops without touching the database cannot be stopped: JavaScriptCore has no public way to interrupt one. `Cmd+.` stops anything that reads, writes or prints, which covers every query. A script silent for 120 seconds is abandoned and the shell restarts. -- Field names that look like integers (`"0"`, `"12"`) sort ahead of the rest in a document literal, which is what JavaScript does with them. +- Field names that are whole numbers up to 4294967294 (`"0"`, `"12"`) sort ahead of the rest in a document literal, which is what JavaScript does with them. A nested object with such a key after another key, or with a key named `__proto__`, refuses the save when it is duplicated or written whole. Use **Insert Document…** or a query for it. ## Troubleshooting diff --git a/project.yml b/project.yml index cffaf9ad63..230a3d1fba 100644 --- a/project.yml +++ b/project.yml @@ -507,6 +507,8 @@ targets: - Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift - Plugins/MongoDBDriverPlugin/MongoDBDecimal128.swift - Plugins/MongoDBDriverPlugin/MongoDBAuthSourceResolver.swift + - Plugins/MongoDBDriverPlugin/MongoDBBinarySubtypes.swift + - Plugins/MongoDBDriverPlugin/MongoDBCapabilities.swift - Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift - Plugins/MongoDBDriverPlugin/MongoDBCollectionSchema.swift - Plugins/MongoDBDriverPlugin/MongoDBCreateDatabasePlan.swift @@ -519,7 +521,11 @@ targets: - Plugins/MongoDBDriverPlugin/MongoDBRawFilterNormalizer.swift - Plugins/MongoDBDriverPlugin/MongoDBSSLMapping.swift - Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift + - Plugins/MongoDBDriverPlugin/MongoDBUpdateDocument.swift + - Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift - Plugins/MongoDBDriverPlugin/MongoDocumentText.swift + - Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift + - Plugins/MongoDBDriverPlugin/MongoNestedValueDiff.swift - Plugins/MongoDBDriverPlugin/MongoDocumentWritePlan.swift - Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift - Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift From 1633a6b60bab6c93a6a90e0d65a65741631d7241 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 26 Sep 2026 20:40:46 +0700 Subject: [PATCH 2/3] fix(plugin-mongodb): refuse grid writes that guess a binary subtype or a mixed field's type, and empty keys insert cannot write --- .../BsonDocumentFlattener.swift | 14 ++ .../MongoDBFieldKinds.swift | 48 ++++ .../MongoDBPluginDriver.swift | 15 ++ .../MongoDBStatementGenerator.swift | 238 +++++++++++++----- .../MongoDBWriteRefusal.swift | 21 +- .../MongoExtendedJsonForm.swift | 9 + TablePro/Resources/Localizable.xcstrings | 18 +- .../Plugins/MongoDBFieldNameUpdateTests.swift | 60 ++++- .../MongoDBNestedValueWriteTests.swift | 153 ++++++++++- .../MongoDBStatementGeneratorTests.swift | 2 +- .../Plugins/MongoDBWriteBackBinaryTests.swift | 90 ++++++- docs/databases/mongodb.mdx | 5 +- project.yml | 1 + 13 files changed, 581 insertions(+), 93 deletions(-) create mode 100644 Plugins/MongoDBDriverPlugin/MongoDBFieldKinds.swift diff --git a/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift b/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift index 7d655ba854..35d4706234 100644 --- a/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift +++ b/Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift @@ -420,6 +420,20 @@ struct BsonDocumentFlattener { return shared } + /// Every kind each top-level field holds in the documents, nulls aside. + static func heldKinds( + in documents: [[String: Any]], + representation: MongoDBUuidRepresentation + ) -> [String: Set] { + var held: [String: Set] = [:] + for doc in documents { + for (field, value) in doc where !(value is NSNull) { + held[field, default: []].insert(valueKind(for: value, representation: representation)) + } + } + return held + } + private static func inferValueKind( for field: String, in documents: [[String: Any]], diff --git a/Plugins/MongoDBDriverPlugin/MongoDBFieldKinds.swift b/Plugins/MongoDBDriverPlugin/MongoDBFieldKinds.swift new file mode 100644 index 0000000000..dbad56f13a --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBFieldKinds.swift @@ -0,0 +1,48 @@ +// +// MongoDBFieldKinds.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// Every kind of value each top-level field held in the documents the grid was handed. +/// +/// A cell shows a document and a string that reads the same as the same text, and the majority +/// kind a column is typed by says nothing about one row. A field that has held documents and no +/// strings shows a document in every cell that opens with `{`; one that has held both cannot say +/// which a cell holds. Added to rather than replaced, like the binary subtypes, so a string read on +/// an earlier page is not forgotten when a later page holds only documents. +struct MongoDBFieldKinds: Sendable { + private var kindsByField: [String: Set] + + static let empty = MongoDBFieldKinds([:]) + + /// Past this many fields a collection records no new ones, and a field it never recorded is + /// treated as unknown rather than as holding one kind. + static let fieldLimit = 10_000 + + init(_ kindsByField: [String: Set]) { + self.kindsByField = kindsByField + } + + static func recording(_ documents: [[String: Any]], representation: MongoDBUuidRepresentation) -> MongoDBFieldKinds { + MongoDBFieldKinds(BsonDocumentFlattener.heldKinds(in: documents, representation: representation)) + } + + var isEmpty: Bool { kindsByField.isEmpty } + + /// The kinds the field was seen holding, or nil when it was never seen. + func kinds(of field: String) -> Set? { + kindsByField[field] + } + + func merging(_ other: MongoDBFieldKinds) -> MongoDBFieldKinds { + var merged = self + for (field, kinds) in other.kindsByField { + guard merged.kindsByField[field] != nil || merged.kindsByField.count < Self.fieldLimit else { continue } + merged.kindsByField[field, default: []].formUnion(kinds) + } + return merged + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index c1f1aef7c9..136c99daa1 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -21,6 +21,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var declaredSchemasByCollection: [String: MongoDBCollectionSchema] = [:] private var identityKindsByCollection: [String: BsonValueKind] = [:] private var binarySubtypesByCollection: [String: MongoDBBinarySubtypes] = [:] + private var fieldKindsByCollection: [String: MongoDBFieldKinds] = [:] private static let logger = Logger(subsystem: "com.TablePro", category: "MongoDBPluginDriver") @@ -821,6 +822,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { columnKinds: columnKindsByCollection[key] ?? [:], declaredKinds: declared.valueKinds, identityKind: identityKindsByCollection[key], + fieldKinds: fieldKindsByCollection[key] ?? .empty, binarySubtypes: binarySubtypesByCollection[key] ?? .empty, declaredBinaryFields: Self.binaryFields(declaredBy: declared), capabilities: { MongoDBCapabilities.parse(self.serverVersion) } @@ -942,6 +944,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { rememberIdentityKind(of: documents, collection: collection) rememberFieldPathKinds(from: documents, collection: collection) rememberBinarySubtypes(of: documents, collection: collection) + rememberFieldKinds(of: documents, collection: collection) let unseen = MongoDBCollectionShape.declaredColumnsMissing(from: sampled, schema: declared) let columns = sampled + unseen let kinds = sampledKinds + unseen.map { declared.field(named: $0)?.valueKind ?? .null } @@ -1019,6 +1022,18 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private static let binarySubtypeLimit = 50_000 + /// Added to rather than replaced, for the same reason as the binary subtypes: a string an earlier + /// page held still makes a text that opens with `{` ambiguous after a later page held none. + private func rememberFieldKinds(of documents: [[String: Any]], collection: String) { + guard !collection.isEmpty else { return } + let page = MongoDBFieldKinds.recording(documents, representation: uuidRepresentation) + guard !page.isEmpty else { return } + let key = columnKindKey(collection) + columnKindLock.withLock { + fieldKindsByCollection[key] = (fieldKindsByCollection[key] ?? .empty).merging(page) + } + } + private func rememberIdentityKind(of documents: [[String: Any]], collection: String) { guard !collection.isEmpty else { return } let kind = BsonDocumentFlattener.uniformKind( diff --git a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index 64b611aae3..72d020671b 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -24,11 +24,14 @@ struct MongoDBStatementGenerator { /// row's own type, and a majority kind would quote an ObjectId in a mostly-string collection and /// match nothing. var identityKind: BsonValueKind? + /// Every kind each field held in the documents read, which says whether a cell showing `{` or + /// `[` holds a document or an array or a string that reads the same. + var fieldKinds: MongoDBFieldKinds = .empty /// The subtype of each binary value the grid shows, so bytes written back keep the subtype they /// were read with. var binarySubtypes: MongoDBBinarySubtypes = .empty - /// Fields the validator declares as binary. New bytes with no earlier value there are generic + /// Fields the validator declares as binary. Bytes typed into a new document there are generic /// binary, subtype 0, which is also what the column's `BLOB` type name says. var declaredBinaryFields: Set = [] /// Asked only for a row that needs `$setField`, since reading the version can wait on the @@ -37,6 +40,25 @@ struct MongoDBStatementGenerator { private static let defaultMarker = "__DEFAULT__" + /// Where a value written whole comes from. A cell carries only text or bytes, so this is all the + /// generator knows about the type the value has to keep. + private enum Provenance { + /// A cell of an existing document, edited from the value it held. Data Rewind puts a value + /// back the same way, so an edit is never taken to be something the user typed. + case edit(replacing: PluginCellValue) + /// A cell of a new document that the user filled in. + case typedIntoNewDocument + /// A cell of a new document copied from another row, as a duplicate or a paste does. + case copiedIntoNewDocument + /// A value of a deleted document, put back as it was read. + case restored + + var replaced: PluginCellValue? { + guard case .edit(let value) = self else { return nil } + return value + } + } + private var collectionAccessor: String { MongoCollectionAccessor.expression(for: collectionName) } @@ -90,18 +112,24 @@ struct MongoDBStatementGenerator { /// NULL and DEFAULT leave a field out of a new document. A row with nothing else in it is the /// empty document, which the server stores with a generated `_id`. + /// + /// A new row's change lists the cells the user filled in; every other value came with the row, + /// copied from the row it duplicates or pastes. private func insertStatement( for change: PluginRowChange, insertedRowData: [Int: [PluginCellValue]] ) throws -> String { + let filledIn = change.cellChanges.map { (field: $0.columnName, value: $0.newValue) } + let typed = Dictionary(filledIn.map { ($0.field, $0.value) }, uniquingKeysWith: { _, last in last }) let cells: [(field: String, value: PluginCellValue)] if let values = insertedRowData[change.rowIndex] { cells = zip(columns, values).map { (field: $0, value: $1) } } else { - cells = change.cellChanges.map { (field: $0.columnName, value: $0.newValue) } + cells = filledIn } let entries = try cells.filter { !isLeftOut($0.value) }.map { cell in - "\(quotedKey(cell.field)): \(try documentValueJson(cell.value, field: cell.field))" + let provenance: Provenance = typed[cell.field] == cell.value ? .typedIntoNewDocument : .copiedIntoNewDocument + return "\(quotedKey(cell.field)): \(try documentValueJson(cell.value, field: cell.field, provenance: provenance))" } return "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" } @@ -113,14 +141,20 @@ struct MongoDBStatementGenerator { /// A value of a whole document the grid writes: a new row, a duplicate or a paste, or a /// deleted document put back. An `_id` the row holds is kept, typed the way the filters type it; /// the grid leaves a new row's `_id` as DEFAULT, so the server generates it. - private func documentValueJson(_ value: PluginCellValue, field: String) throws -> String { - if field == MongoDBCollectionDDL.idField { - return try idValueJson(value) - } - guard !field.isEmpty, field != "__proto__" else { - throw MongoDBWriteRefusal.unwritableFieldName(field: field) + /// + /// The shell inserts through libmongoc, which refuses a document holding an empty key at any + /// depth, and a JavaScript object reads `__proto__` as its prototype and drops it. An update + /// carries both, so only a new document refuses them, before anything in the save is sent. + private func documentValueJson(_ value: PluginCellValue, field: String, provenance: Provenance) throws -> String { + guard !field.isEmpty else { throw MongoDBWriteRefusal.emptyFieldNameInNewDocument } + guard field != "__proto__" else { throw MongoDBWriteRefusal.prototypeFieldInNewDocument } + let json = field == MongoDBCollectionDDL.idField + ? try idValueJson(value) + : try valueJson(value, field: field, provenance: provenance) + if opensContainer(json), case .text(let text) = value, parsedContainer(text)?.holdsEmptyKey == true { + throw MongoDBWriteRefusal.emptyKeyInNewDocument(field: field) } - return try valueJson(value, field: field, replacing: nil) + return json } // MARK: - Restore @@ -136,11 +170,12 @@ struct MongoDBStatementGenerator { do { return try rows.map { row in guard idIndex < row.count else { throw MongoDBWriteRefusal.missingIdentity } - var entries = ["\"_id\": \(try idValueJson(row[idIndex]))"] + let idField = MongoDBCollectionDDL.idField + var entries = ["\(quotedKey(idField)): \(try documentValueJson(row[idIndex], field: idField, provenance: .restored))"] for (index, value) in row.enumerated() where index != idIndex && index < columns.count { guard !isLeftOut(value) else { continue } let field = columns[index] - entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field))") + entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field, provenance: .restored))") } let statement = "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" return (statement: statement, parameters: []) @@ -185,29 +220,30 @@ struct MongoDBStatementGenerator { case .null: return .remove(field: field) case .bytes(let data): - return .whole(field: field, json: try binaryJson(data, field: field, replacing: oldValue)) + return .whole(field: field, json: try binaryJson(data, field: field, provenance: .edit(replacing: oldValue))) case .text(let text): guard text != Self.defaultMarker else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } if let nested = try nestedEdit(of: field, from: oldValue, to: text) { return nested } - return .whole(field: field, json: try valueJson(newValue, field: field, replacing: oldValue)) + return .whole(field: field, json: try valueJson(newValue, field: field, provenance: .edit(replacing: oldValue))) } } /// An edit of a nested document or array, as the paths it changed. /// - /// Nil when the cell does not hold a complete one on both sides, and the edit is then written - /// whole. An old value shortened for display cannot be diffed, but a complete value typed over - /// it replaces the field without needing it; new text that is still shortened is refused when - /// it is written. + /// Nil when the cell is not known to hold a complete one, and the edit is then written whole. A + /// path into a string is refused by the server, so a field that also held strings is never + /// diffed, whatever most of its rows hold. An old value shortened for display cannot be diffed, + /// but a complete value typed over it replaces the field without needing it; new text that is + /// still shortened is refused when it is written. private func nestedEdit(of field: String, from oldValue: PluginCellValue, to text: String) throws -> CellWrite? { - guard isContainerKind(field), case .text(let oldText) = oldValue, opensContainer(text), - !JSONTruncation.isIncompleteStructure(text), !JSONTruncation.isIncompleteStructure(oldText), - let old = try? MongoDocumentText.Value(parsing: oldText), old.isContainer else { + guard opensContainer(text), !JSONTruncation.isIncompleteStructure(text), + holdsContainer(oldValue, field: field), case .text(let oldText) = oldValue, + !JSONTruncation.isIncompleteStructure(oldText), let old = parsedContainer(oldText) else { return nil } - guard let edited = try? MongoDocumentText.Value(parsing: text), edited.isContainer else { + guard let edited = parsedContainer(text) else { throw MongoDBWriteRefusal.unreadableJSON(field: field) } let changes = MongoNestedValueDiff.changes(from: old, to: edited, at: field) @@ -290,37 +326,39 @@ struct MongoDBStatementGenerator { } /// A cell's value as the JSON the statement carries. - private func valueJson(_ value: PluginCellValue, field: String, replacing oldValue: PluginCellValue?) throws -> String { + private func valueJson(_ value: PluginCellValue, field: String, provenance: Provenance) throws -> String { switch value { case .null: return "null" case .bytes(let data): - return try binaryJson(data, field: field, replacing: oldValue) + return try binaryJson(data, field: field, provenance: provenance) case .text(let text): - if case .bytes(let oldData) = oldValue { + if case .edit(replacing: .bytes(let oldData)) = provenance { return try textIntoBinaryJson(text, field: field, replacing: oldData) } guard !JSONTruncation.isIncompleteStructure(text) else { throw MongoDBWriteRefusal.truncatedValue(field: field) } - return try jsonValue(for: text, field: field) + return try jsonValue(for: text, field: field, provenance: provenance) } } - private func binaryJson(_ data: Data, field: String, replacing oldValue: PluginCellValue?) throws -> String { - let subtype = try binarySubtype(of: data, field: field, replacing: oldValue) + private func binaryJson(_ data: Data, field: String, provenance: Provenance) throws -> String { + let subtype = try binarySubtype(of: data, field: field, provenance: provenance) return MongoDBUuidCodec.extendedJson(for: MongoDBBinaryValue(data: data, subtype: subtype)) } - /// Edited bytes keep the subtype of the value they replace. Copied bytes keep the subtype they - /// were read with. Bytes with neither are generic binary only where the validator declares the - /// field binary; anywhere else the subtype is unknown and the write is refused. - private func binarySubtype(of data: Data, field: String, replacing oldValue: PluginCellValue?) throws -> UInt8 { - if case .bytes(let oldData) = oldValue { + /// Edited bytes keep the subtype of the value they replace, and any other bytes the subtype + /// they were read with. A subtype nothing recorded is never assumed: a restore after a relaunch + /// hands back bytes alone, and so does an edit over a value that was not binary, which is how + /// Data Rewind puts bytes back. Only bytes the user typed into a new document are new, and they + /// are generic binary where the validator declares the field binary. + private func binarySubtype(of data: Data, field: String, provenance: Provenance) throws -> UInt8 { + if case .edit(replacing: .bytes(let oldData)) = provenance { return try onlySubtype(binarySubtypes.subtypes(of: oldData, in: field), field: field) } let known = binarySubtypes.subtypes(of: data, in: field) - if known.isEmpty, declaredBinaryFields.contains(field) { + if known.isEmpty, case .typedIntoNewDocument = provenance, declaredBinaryFields.contains(field) { return 0 } return try onlySubtype(known, field: field) @@ -340,41 +378,40 @@ struct MongoDBStatementGenerator { return wrapped } guard text.isEmpty else { throw MongoDBWriteRefusal.binaryNeedsBytes(field: field) } - return try binaryJson(Data(), field: field, replacing: .bytes(oldData)) + return try binaryJson(Data(), field: field, provenance: .edit(replacing: .bytes(oldData))) } private func shellJson(_ value: MongoDocumentText.Value, field: String) throws -> String { try MongoExtendedJsonForm.shellValue(value, field: field).compactText } - private func isContainerKind(_ field: String) -> Bool { - let kind = kind(of: field) - return kind == .document || kind == .array - } - private func opensContainer(_ text: String) -> Bool { text.hasPrefix("{") || text.hasPrefix("[") } + private func parsedContainer(_ text: String) -> MongoDocumentText.Value? { + guard let parsed = try? MongoDocumentText.Value(parsing: text), parsed.isContainer else { return nil } + return parsed + } + private func quotedKey(_ field: String) -> String { "\"\(escapeJsonString(field))\"" } // MARK: - Identity - /// An `_id` held as bytes filters on binary with the subtype it was read with, or the one every - /// sampled `_id` shares. + /// An `_id` held as bytes filters on binary with the subtype it was read with. The same bytes + /// under another subtype are another `_id`, which may name another document, so a subtype that + /// was not read for these bytes is never borrowed from the other rows. private func idValueJson(_ value: PluginCellValue) throws -> String { - let idField = MongoDBCollectionDDL.idField switch value { case .null: throw MongoDBWriteRefusal.missingIdentity case .bytes(let data): - let known = binarySubtypes.subtypes(of: data, in: idField) - if known.isEmpty, case .binary(let subtype) = identityKind { - return MongoDBUuidCodec.extendedJson(for: MongoDBBinaryValue(data: data, subtype: subtype)) + let subtypes = binarySubtypes.subtypes(of: data, in: MongoDBCollectionDDL.idField) + guard subtypes.count == 1, let subtype = subtypes.first else { + throw MongoDBWriteRefusal.identitySubtypeUnknown } - let subtype = try onlySubtype(known, field: idField) return MongoDBUuidCodec.extendedJson(for: MongoDBBinaryValue(data: data, subtype: subtype)) case .text(let text): if let document = try documentIdJson(text) { @@ -436,7 +473,7 @@ struct MongoDBStatementGenerator { /// The statement is JavaScript the shell evaluates, so text is only ever pasted in when it is /// strict JSON. A stored string that merely starts with `[` and ends with `]` would otherwise run /// as code the moment its row is duplicated or its delete is undone. - private func jsonValue(for value: String, field: String) throws -> String { + private func jsonValue(for value: String, field: String, provenance: Provenance) throws -> String { if declaredKinds[field] == .string { return "\"\(escapeJsonString(value))\"" } @@ -452,7 +489,7 @@ struct MongoDBStatementGenerator { if let binary = MongoDBUuidCodec.extendedJsonFromWrapper(value) { return binary } - if let container = try containerJson(value, field: field) { + if let container = try containerJson(value, field: field, provenance: provenance) { return container } return "\"\(escapeJsonString(value))\"" @@ -493,17 +530,102 @@ struct MongoDBStatementGenerator { private static let largestExactDouble: UInt64 = 1 << 53 - /// A nested document or array, spelled so the shell stores the types its text shows. Text that - /// only looks like one is a string, except in a field that holds documents or arrays, where it - /// is refused rather than stored as a string. - private func containerJson(_ value: String, field: String) throws -> String? { + /// A nested document or array, spelled so the shell stores the types its text shows, or nil for + /// text that is written as a string. + private func containerJson(_ value: String, field: String, provenance: Provenance) throws -> String? { guard opensContainer(value) else { return nil } - guard value.hasSuffix("}") || value.hasSuffix("]"), - let parsed = try? MongoDocumentText.Value(parsing: value), parsed.isContainer else { - if isContainerKind(field) { throw MongoDBWriteRefusal.unreadableJSON(field: field) } - return nil + let parsed = value.hasSuffix("}") || value.hasSuffix("]") ? parsedContainer(value) : nil + switch containerReading(of: field, isJSON: parsed != nil, provenance: provenance) { + case .container: + guard let parsed else { throw MongoDBWriteRefusal.unreadableJSON(field: field) } + return try shellJson(parsed, field: field) + case .jsonOrString: + return try parsed.map { try shellJson($0, field: field) } + case .ambiguous: + throw MongoDBWriteRefusal.documentOrText(field: field) + } + } + + // MARK: - Documents and arrays + + /// What text opening with `{` or `[` is written as. + private enum ContainerReading { + /// A document or an array, and text that is not JSON is refused rather than stored as a string. + case container + /// JSON is a document or an array, and any other text a string. + case jsonOrString + /// The field holds both, and nothing says which this value is meant to be. + case ambiguous + } + + /// A cell that held a document or an array stays one. Otherwise the kinds the field held decide: + /// JSON is a document or an array where the field held those and no strings, and could be either + /// where it held both, which is refused. Text that is not JSON can only be a string when it was + /// read from a cell, where a document always shows as JSON, or typed over a value that was not + /// one. An existing value whose field was never read says nothing either way. + private func containerReading(of field: String, isJSON: Bool, provenance: Provenance) -> ContainerReading { + if let replaced = provenance.replaced, holdsContainer(replaced, field: field) { + return .container } - return try shellJson(parsed, field: field) + guard let held = heldKinds(of: field) else { + return isJSON && standsForUnreadValue(provenance) ? .ambiguous : .jsonOrString + } + guard held.contains(.document) || held.contains(.array) else { return .jsonOrString } + guard held.contains(.string) else { return .container } + return !isJSON && nonJSONCanOnlyBeText(provenance) ? .jsonOrString : .ambiguous + } + + /// Whether a cell holds a document or an array: text of that shape, in a field that held that + /// shape and no strings, since a string can read the same. + private func holdsContainer(_ value: PluginCellValue, field: String) -> Bool { + guard case .text(let text) = value, opensContainer(text), let held = heldKinds(of: field) else { return false } + let shape: BsonValueKind = text.hasPrefix("[") ? .array : .document + return held.contains(shape) && !held.contains(.string) + } + + private func nonJSONCanOnlyBeText(_ provenance: Provenance) -> Bool { + switch provenance { + case .restored, .copiedIntoNewDocument: + return true + case .edit(let replaced): + return holdsOtherThanContainer(replaced) + case .typedIntoNewDocument: + return false + } + } + + /// Whether a cell is known to hold something other than a document or an array: text that does + /// not open one, or opens one and is not JSON, which only a string can be. + private func holdsOtherThanContainer(_ value: PluginCellValue) -> Bool { + switch value { + case .null: + return false + case .bytes: + return true + case .text(let text): + guard opensContainer(text) else { return true } + return !JSONTruncation.isIncompleteStructure(text) && parsedContainer(text) == nil + } + } + + /// A value a document already holds that reads like a document or an array: one put back, or + /// the one an edit replaces. + private func standsForUnreadValue(_ provenance: Provenance) -> Bool { + switch provenance { + case .restored: + return true + case .edit(replacing: .text(let text)): + return opensContainer(text) + case .edit, .typedIntoNewDocument, .copiedIntoNewDocument: + return false + } + } + + /// The validator's kind where it declares one, which the server enforces, otherwise every kind + /// the documents read held. + private func heldKinds(of field: String) -> Set? { + if let declared = declaredKinds[field] { return [declared] } + return fieldKinds.kinds(of: field) } /// Escape special characters for JSON strings (handles Unicode control chars U+0000-U+001F) diff --git a/Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift b/Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift index 2bb3a709eb..bda65c8dad 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBWriteRefusal.swift @@ -15,14 +15,18 @@ enum MongoDBWriteRefusal: Error, Equatable { case binarySubtypeUnknown(field: String) case binaryNeedsBytes(field: String) case truncatedValue(field: String) - case unwritableFieldName(field: String) + case emptyFieldNameInNewDocument + case emptyKeyInNewDocument(field: String) + case prototypeFieldInNewDocument case fieldNeedsMongoDB5(field: String) case unreadableJSON(field: String) + case documentOrText(field: String) case integerTooLarge(field: String) case reorderedByTheShell(field: String) case noDefaultValue(field: String) case identityChanged case missingIdentity + case identitySubtypeUnknown var reason: String { switch self { @@ -41,11 +45,15 @@ enum MongoDBWriteRefusal: Error, Equatable { format: String(localized: "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query."), field ) - case .unwritableFieldName(let field): + case .emptyFieldNameInNewDocument: + return String(localized: "The MongoDB client library will not insert a field whose name is empty. Set it to NULL, then set it on the saved document.") + case .emptyKeyInNewDocument(let field): return String( - format: String(localized: "The shell cannot write a field named \u{201C}%@\u{201D} into a new document. Insert this document with a query."), + format: String(localized: "%@ holds an empty key, which the MongoDB client library will not insert. Set it to NULL, then set it on the saved document."), field ) + case .prototypeFieldInNewDocument: + return String(localized: "The shell drops a field named __proto__ from a new document. Set it to NULL, then set it on the saved document.") case .fieldNeedsMongoDB5(let field): return String( format: String(localized: "A field named \u{201C}%@\u{201D} can only be changed on MongoDB 5.0 or later, which can address a dot or a leading $ in a name."), @@ -56,6 +64,11 @@ enum MongoDBWriteRefusal: Error, Equatable { format: String(localized: "%@ holds a document or an array, and this text is not valid JSON."), field ) + case .documentOrText(let field): + return String( + format: String(localized: "%@ holds text in some rows and documents or arrays in others, so this value could be either. Change it with a query."), + field + ) case .integerTooLarge(let field): return String( format: String(localized: "A number in %@ is larger than a 64-bit integer. Write it as a decimal, such as {\"$numberDecimal\": \"12345678901234567890\"}."), @@ -75,6 +88,8 @@ enum MongoDBWriteRefusal: Error, Equatable { return String(localized: "MongoDB does not let a document's _id change. Duplicate the row with the new _id, then delete this one.") case .missingIdentity: return String(localized: "This row has no _id, so TablePro cannot tell which document to change.") + case .identitySubtypeUnknown: + return String(localized: "This row's _id is binary data whose subtype is not known, so TablePro cannot tell which document it names.") } } diff --git a/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift b/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift index 6a4a7023f2..9003bb0307 100644 --- a/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift +++ b/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift @@ -203,4 +203,13 @@ extension MongoDocumentText.Value { case .string, .number, .literal: return false } } + + /// Whether any object in the value, at any depth, has a key that is the empty string. + var holdsEmptyKey: Bool { + switch self { + case .object(let members): return members.contains { $0.key.isEmpty || $0.value.holdsEmptyKey } + case .array(let elements): return elements.contains { $0.holdsEmptyKey } + case .string, .number, .literal: return false + } + } } diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 54e76d9ee2..ed906a32e1 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -184453,9 +184453,6 @@ }, "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query." : { - }, - "The shell cannot write a field named “%@” into a new document. Insert this document with a query." : { - }, "%@ holds a document or an array, and this text is not valid JSON." : { @@ -184480,6 +184477,21 @@ }, "The shell would reorder or drop a key inside %@: it lists number-like keys first and drops __proto__. Change this field with a query." : { + }, + "This row's _id is binary data whose subtype is not known, so TablePro cannot tell which document it names." : { + + }, + "The MongoDB client library will not insert a field whose name is empty. Set it to NULL, then set it on the saved document." : { + + }, + "%@ holds an empty key, which the MongoDB client library will not insert. Set it to NULL, then set it on the saved document." : { + + }, + "The shell drops a field named __proto__ from a new document. Set it to NULL, then set it on the saved document." : { + + }, + "%@ holds text in some rows and documents or arrays in others, so this value could be either. Change it with a query." : { + } }, "version" : "1.1" diff --git a/TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift b/TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift index 0966e41ac3..6bc111f63d 100644 --- a/TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift +++ b/TableProTests/Plugins/MongoDBFieldNameUpdateTests.swift @@ -21,6 +21,7 @@ struct MongoDBFieldNameUpdateTests { collectionName: "items", columns: columns, columnKinds: kinds, + fieldKinds: MongoDBFieldKinds(kinds.mapValues { [$0] }), capabilities: { MongoDBCapabilities.parse(version) } ) let change = PluginRowChange( @@ -178,21 +179,58 @@ struct MongoDBFieldNameUpdateTests { // MARK: - Inserts - @Test("A new document cannot hold __proto__ or an empty name, which the shell cannot write") - func insertRefusesUnwritableNames() { - for name in ["__proto__", ""] { - let gen = MongoDBStatementGenerator(collectionName: "items", columns: ["_id", name]) - #expect(throws: MongoDBWriteRefusal.unwritableFieldName(field: name).refusal(ofRow: 0), "\(name)") { - try gen.generateRowWrites( - from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], - insertedRowData: [0: [nil, "x"]], - deletedRowIndices: [], - insertedRowIndices: [0] - ) + private func insert(_ values: [PluginCellValue], columns: [String]) throws -> [PluginRowWrite] { + try MongoDBStatementGenerator(collectionName: "items", columns: columns).generateRowWrites( + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + insertedRowData: [0: values], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + } + + /// Measured on 7.0.43 through the shell: insertOne and insertMany fail with "[22] invalid + /// document for insert: empty key" for an empty name at the top level or inside a value, while a + /// classic $set of a value holding one and $setField of an empty name both succeed. + @Test("A new document with an empty field name is refused, since libmongoc will not insert it") + func insertRefusesAnEmptyName() { + #expect(throws: MongoDBWriteRefusal.emptyFieldNameInNewDocument.refusal(ofRow: 0)) { + try insert([nil, "x"], columns: ["_id", ""]) + } + } + + @Test("A new document holding an empty key inside a value is refused at any depth") + func insertRefusesANestedEmptyKey() { + for nested in [#"{"":5}"#, #"{"a":[{"b":1,"":2}]}"#] { + #expect(throws: MongoDBWriteRefusal.emptyKeyInNewDocument(field: "m").refusal(ofRow: 0), "\(nested)") { + try insert([nil, .text(nested)], columns: ["_id", "m"]) } } } + @Test("A restored document holding an empty key is refused rather than failing in the shell") + func restoreRefusesAnEmptyKey() { + let gen = MongoDBStatementGenerator( + collectionName: "items", columns: ["_id", "", "m"], fieldKinds: MongoDBFieldKinds(["m": [.document]]) + ) + + #expect(gen.generateRestore(rows: [["507f1f77bcf86cd799439011", "x", nil]]) == nil) + #expect(gen.generateRestore(rows: [["507f1f77bcf86cd799439011", nil, #"{"":5}"#]]) == nil) + } + + @Test("An update carries a value holding an empty key, which the shell writes") + func updateCarriesANestedEmptyKey() throws { + let statement = try update([("m", nil, #"{"":6}"#)], columns: ["_id", "m"]) + + #expect(statement == #"db.items.updateOne({"_id": 1}, {"$set": {"m": {"":6}}})"#) + } + + @Test("A new document cannot hold __proto__, which the shell reads as the prototype and drops") + func insertRefusesPrototype() { + #expect(throws: MongoDBWriteRefusal.prototypeFieldInNewDocument.refusal(ofRow: 0)) { + try insert([nil, "x"], columns: ["_id", "__proto__"]) + } + } + @Test("A new document keeps a dotted or $ name as a literal key, which insert accepts") func insertKeepsDottedNames() throws { let gen = MongoDBStatementGenerator(collectionName: "items", columns: ["_id", "k.d", "$q"]) diff --git a/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift b/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift index 832dc91d5b..85d401e756 100644 --- a/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift +++ b/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift @@ -19,21 +19,43 @@ struct MongoDBNestedValueWriteTests { to newText: String, kind: BsonValueKind = .document ) throws -> String { - let gen = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", column], columnKinds: [column: kind]) + try update(column, from: .text(oldText), to: newText, held: [kind], majority: kind) + } + + /// An edit in a field whose documents held `held`, or were never read when it is nil, typed by + /// its `majority` kind. + private func update( + _ column: String, + from oldValue: PluginCellValue, + to newText: String, + held: Set?, + majority: BsonValueKind = .document + ) throws -> String { + let gen = MongoDBStatementGenerator( + collectionName: "people", + columns: ["_id", column], + columnKinds: [column: majority], + fieldKinds: MongoDBFieldKinds(held.map { [column: $0] } ?? [:]) + ) let change = PluginRowChange( rowIndex: 0, type: .update, - cellChanges: [(columnIndex: 1, columnName: column, oldValue: .text(oldText), newValue: .text(newText))], - originalRow: [.text("1"), .text(oldText)] + cellChanges: [(columnIndex: 1, columnName: column, oldValue: oldValue, newValue: .text(newText))], + originalRow: [.text("1"), oldValue] ) let writes = try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []) return try #require(writes.first?.statement) } - private func insert(_ column: String, _ text: String) throws -> String { - let gen = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", column]) + private func insert(_ column: String, _ text: String, held: Set? = nil, typed: Bool = false) throws -> String { + let gen = MongoDBStatementGenerator( + collectionName: "people", + columns: ["_id", column], + fieldKinds: MongoDBFieldKinds(held.map { [column: $0] } ?? [:]) + ) + let filledIn = typed ? [(columnIndex: 1, columnName: column, oldValue: PluginCellValue.null, newValue: PluginCellValue.text(text))] : [] let writes = try gen.generateRowWrites( - from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: filledIn, originalRow: nil)], insertedRowData: [0: [nil, .text(text)]], deletedRowIndices: [], insertedRowIndices: [0] @@ -184,6 +206,125 @@ struct MongoDBNestedValueWriteTests { } } + // MARK: - A field that held documents and strings + + /// Measured on 7.0.43: in a field where most rows hold documents, `{"$set": {"f.a": 2}}` on the + /// row holding the string `{"a":1}` fails with "[28] Cannot create field 'a' in element". + @Test("A string that reads as JSON, in a field of mostly documents, is never diffed into a path") + func jsonLookingStringInAMixedFieldIsRefused() { + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0)) { + try update("f", from: .text(#"{"a":1}"#), to: #"{"a":2}"#, held: [.document, .string]) + } + } + + @Test("A document in a field that also held strings is written whole or not at all, never as a path") + func documentInAMixedFieldIsNeverAPath() throws { + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0)) { + try update("f", from: .text(#"{"a":1,"b":2}"#), to: #"{"a":5,"b":2}"#, held: [.document, .string]) + } + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0)) { + try update("f", from: .text("[1,2]"), to: "[1,3]", held: [.array, .string], majority: .array) + } + } + + @Test("Text that is not JSON typed over a string that is not JSON either stays a string") + func malformedStringInAMixedFieldStaysAString() throws { + let statement = try update("f", from: .text("{abc"), to: "{abcd", held: [.document, .string]) + + #expect(statement == #"db.people.updateOne({"_id": 1}, {"$set": {"f": "{abcd"}})"#) + } + + @Test("JSON typed over a string in a field that also held documents is refused, since it could be either") + func jsonOverAStringInAMixedFieldIsRefused() { + for old: PluginCellValue in [.text("{abc"), .text("plain"), .null] { + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0), "\(old)") { + try update("f", from: old, to: #"{"x":1}"#, held: [.document, .string]) + } + } + } + + @Test("Text that is not JSON typed over a document in a mixed field is refused, not stored as a string") + func malformedTextOverAPossibleDocumentIsRefused() { + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0)) { + try update("f", from: .text(#"{"a":1}"#), to: #"{"a":"#, held: [.document, .string]) + } + } + + @Test("A document and an array in one field are told apart by their text, so each is still diffed") + func documentsAndArraysAreToldApart() throws { + let document = try update("f", from: .text(#"{"a":1,"b":2}"#), to: #"{"a":5,"b":2}"#, held: [.document, .array]) + let array = try update("f", from: .text("[1,2]"), to: "[1,3]", held: [.document, .array]) + + #expect(document == #"db.people.updateOne({"_id": 1}, {"$set": {"f.a": 5}})"#) + #expect(array == #"db.people.updateOne({"_id": 1}, {"$set": {"f.1": 3}})"#) + } + + @Test("A field whose documents were never read is never diffed") + func unreadFieldIsNeverAPath() { + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0)) { + try update("f", from: .text(#"{"a":1}"#), to: #"{"a":2}"#, held: nil) + } + } + + @Test("A copied value that reads as JSON in a mixed field is refused, and one that does not is a string") + func copiedValuesInAMixedField() throws { + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0)) { + try insert("f", #"{"a":1}"#, held: [.document, .string]) + } + #expect(throws: MongoDBWriteRefusal.documentOrText(field: "f").refusal(ofRow: 0)) { + try insert("f", "{abc", held: [.document, .string], typed: true) + } + let copied = try insert("f", "{abc", held: [.document, .string]) + #expect(copied == #"db.people.insertOne({"f": "{abc"})"#) + } + + @Test("A restore that reads as JSON in a field never read is refused, and one that does not is a string") + func restoreIntoAnUnreadField() throws { + let relaunched = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", "f"]) + let documents = MongoDBStatementGenerator( + collectionName: "people", columns: ["_id", "f"], fieldKinds: MongoDBFieldKinds(["f": [.document]]) + ) + + #expect(relaunched.generateRestore(rows: [["507f1f77bcf86cd799439011", #"{"a":1}"#]]) == nil) + let text = try #require(relaunched.generateRestore(rows: [["507f1f77bcf86cd799439011", "{abc"]])?.first) + #expect(text.statement.contains(#""f": "{abc""#)) + let document = try #require(documents.generateRestore(rows: [["507f1f77bcf86cd799439011", #"{"a":1}"#]])?.first) + #expect(document.statement.contains(#""f": {"a":1}"#)) + } + + // MARK: - Recording what each field held + + @Test("Each field records every kind it held, nulls aside") + func fieldKindsRecordEveryKind() { + let recorded = MongoDBFieldKinds.recording( + [["f": ["a": 1], "g": "x", "n": NSNull()], ["f": "{\"a\":1}", "g": "y"], ["f": [1]]], + representation: .unspecified + ) + + #expect(recorded.kinds(of: "f") == [.document, .string, .array]) + #expect(recorded.kinds(of: "g") == [.string]) + #expect(recorded.kinds(of: "n") == nil) + #expect(recorded.kinds(of: "missing") == nil) + } + + @Test("A later page adds to what a field held, so a string read earlier is not forgotten") + func fieldKindsMergeRatherThanReplace() { + let merged = MongoDBFieldKinds(["f": [.string]]).merging(MongoDBFieldKinds(["f": [.document], "g": [.int32]])) + + #expect(merged.kinds(of: "f") == [.string, .document]) + #expect(merged.kinds(of: "g") == [.int32]) + } + + @Test("Past the field limit no new field is recorded, and an unrecorded field reads as unknown") + func fieldKindsStopAtTheLimit() { + let full = MongoDBFieldKinds(Dictionary(uniqueKeysWithValues: (0 ..< MongoDBFieldKinds.fieldLimit).map { ("k\($0)", Set([BsonValueKind.string])) })) + + let merged = full.merging(MongoDBFieldKinds(["k0": [.document], "new": [.document]])) + + #expect(merged.kinds(of: "k0") == [.string, .document]) + #expect(merged.kinds(of: "new") == nil) + } + // MARK: - Whole values @Test("A whole value keeps a whole-number double as a double and a 64-bit integer as one") diff --git a/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift b/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift index f51a112f43..569d47f349 100644 --- a/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift +++ b/TableProTests/Plugins/MongoDBStatementGeneratorTests.swift @@ -1008,7 +1008,7 @@ struct MongoDBStatementGeneratorTests { originalRow: [.bytes(Data([0x01, 0x02])), .text("Alice")] ) - #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "_id").refusal(ofRow: 0)) { + #expect(throws: MongoDBWriteRefusal.identitySubtypeUnknown.refusal(ofRow: 0)) { try gen.generateRowWrites(from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: []) } } diff --git a/TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift b/TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift index f439b22ba1..63f8bbbaec 100644 --- a/TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift +++ b/TableProTests/Plugins/MongoDBWriteBackBinaryTests.swift @@ -35,9 +35,18 @@ struct MongoDBWriteBackBinaryTests { return recorded } - private func insert(_ values: [PluginCellValue], with gen: MongoDBStatementGenerator) throws -> [PluginRowWrite] { - try gen.generateRowWrites( - from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil)], + /// A new row whose `typed` columns the user filled in; every other value was copied into it. + private func insert( + _ values: [PluginCellValue], + typed: Set = [], + columns: [String] = ["_id", "name", "thumbnail"], + with gen: MongoDBStatementGenerator + ) throws -> [PluginRowWrite] { + let filledIn = typed.sorted().map { index in + (columnIndex: index, columnName: columns[index], oldValue: PluginCellValue.null, newValue: values[index]) + } + return try gen.generateRowWrites( + from: [PluginRowChange(rowIndex: 0, type: .insert, cellChanges: filledIn, originalRow: nil)], insertedRowData: [0: values], deletedRowIndices: [], insertedRowIndices: [0] @@ -97,15 +106,33 @@ struct MongoDBWriteBackBinaryTests { } } - @Test("New bytes in a field the validator declares binary are generic binary") + @Test("Bytes typed into a new document, in a field the validator declares binary, are generic binary") func declaredBinaryFieldTakesSubtypeZero() throws { let gen = generator(declaredBinary: ["thumbnail"]) - let writes = try insert(["__DEFAULT__", "a", .bytes(Self.png)], with: gen) + let writes = try insert(["__DEFAULT__", "a", .bytes(Self.png)], typed: [2], with: gen) #expect(writes[0].statement.contains(#""subType": "00""#)) } + @Test("Bytes copied into a new document, as a duplicate or a paste does, never take a default subtype") + func copiedBytesInADeclaredFieldAreRefused() { + let gen = generator(declaredBinary: ["thumbnail"]) + + #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "thumbnail").refusal(ofRow: 0)) { + try insert(["__DEFAULT__", "a", .bytes(Self.png)], typed: [1], with: gen) + } + } + + @Test("Bytes typed into a new document keep a subtype that was read for them") + func typedBytesKeepARecordedSubtype() throws { + let gen = generator(subtypes: subtypes([(Self.png, 5, "thumbnail")]), declaredBinary: ["thumbnail"]) + + let writes = try insert(["__DEFAULT__", "a", .bytes(Self.png)], typed: [2], with: gen) + + #expect(writes[0].statement.contains(#""subType": "05""#)) + } + @Test("A row empty apart from a value that cannot be written is refused, not inserted as {}") func refusedValueNeverBecomesEmptyDocument() { #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "thumbnail").refusal(ofRow: 0)) { @@ -161,6 +188,47 @@ struct MongoDBWriteBackBinaryTests { } } + @Test("Bytes put back over a missing value keep the subtype they were read with") + func bytesOverNullKeepTheirRecordedSubtype() throws { + let gen = generator(subtypes: subtypes([(Self.signature, 5, "thumbnail")]), declaredBinary: ["thumbnail"]) + + let writes = try edit("thumbnail", from: .null, to: .bytes(Self.signature), with: gen) + + #expect(writes[0].statement.contains(#""thumbnail": {"$binary": {"base64": "AAECAw==", "subType": "05"}}"#)) + } + + /// Data Rewind undoes Set NULL on a binary field with this very edit, holding bytes alone, so a + /// generator that has not read them since a relaunch cannot tell them from bytes the user typed. + @Test("Bytes set over a missing value with no recorded subtype are refused, even in a declared binary field") + func bytesOverNullWithoutASubtypeAreRefused() { + let relaunched = generator(declaredBinary: ["thumbnail"]) + + #expect(throws: MongoDBWriteRefusal.binarySubtypeUnknown(field: "thumbnail").refusal(ofRow: 0)) { + try edit("thumbnail", from: .null, to: .bytes(Self.signature), with: relaunched) + } + } + + // MARK: - Restore + + @Test("A restore keeps the subtype its bytes were read with") + func restoreKeepsTheRecordedSubtype() throws { + let gen = generator(subtypes: subtypes([(Self.signature, 5, "thumbnail")]), declaredBinary: ["thumbnail"]) + + let restored = try #require(gen.generateRestore(rows: [[.text(Self.objectId), "a", .bytes(Self.signature)]])?.first) + + #expect(restored.statement.contains(#""thumbnail": {"$binary": {"base64": "AAECAw==", "subType": "05"}}"#)) + } + + /// After a relaunch the driver is new and its registry empty, and a Data Rewind record holds the + /// bytes without their subtype. Subtype 0 in a declared binary field used to fill the gap, which + /// stored a subtype-5 value as subtype 0 and reported the restore as done. + @Test("A restore by a new generator with an empty registry refuses bytes rather than assuming a subtype") + func restoreAfterARelaunchRefusesBytes() { + let relaunched = generator(declaredBinary: ["thumbnail"]) + + #expect(relaunched.generateRestore(rows: [[.text(Self.objectId), "a", .bytes(Self.signature)]]) == nil) + } + @Test("DEFAULT is refused on an update, because MongoDB has no default values") func defaultMarkerIsRefused() { #expect(throws: MongoDBWriteRefusal.noDefaultValue(field: "thumbnail").refusal(ofRow: 0)) { @@ -194,14 +262,16 @@ struct MongoDBWriteBackBinaryTests { #expect(writes.map { $0.rowIndices } == [[0], [1]]) } - @Test("A binary _id with no recorded subtype uses the subtype every sampled _id shares") - func binaryIdFallsBackToTheSharedSubtype() throws { + /// The same bytes under another subtype are another `_id`, so a subtype borrowed from the other + /// rows could delete a different document. + @Test("A binary _id with no recorded subtype is refused, even when every sampled _id shares one") + func binaryIdWithoutARecordedSubtypeIsRefused() { let gen = generator(columns: ["_id", "name"], identityKind: .binary(subtype: 3)) let delete = PluginRowChange(rowIndex: 0, type: .delete, cellChanges: [], originalRow: [.bytes(Data([1])), "k"]) - let writes = try gen.generateRowWrites(from: [delete], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: []) - - #expect(writes.first?.statement.contains(#""subType": "03""#) == true) + #expect(throws: MongoDBWriteRefusal.identitySubtypeUnknown.refusal(ofRow: 0)) { + try gen.generateRowWrites(from: [delete], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: []) + } } // MARK: - Recording diff --git a/docs/databases/mongodb.mdx b/docs/databases/mongodb.mdx index 8456a1955c..eaf3ca39bd 100644 --- a/docs/databases/mongodb.mdx +++ b/docs/databases/mongodb.mdx @@ -214,7 +214,10 @@ New connections default to **Disabled**, and the driver has no TLS fallback: **P ## Limitations - A row with no `_id` cannot be updated or deleted. The save is refused rather than matched on the remaining fields, and every change stays pending. Keep `_id` in the projection so every row carries one. -- A binary value keeps the subtype it was read with. New bytes in a field whose validator does not declare `binData`, or the same bytes read with two different subtypes, refuse the save: write that value with a query. +- A binary value keeps the subtype it was read with. Bytes typed into a new row are saved as subtype 0 where the validator declares the field `binData`. +- Other bytes refuse the save when their subtype was never read, or was read as two different ones: bytes typed into an existing document, pasted from another collection, or put back by Data Rewind after TablePro restarts. Write that value with a query. +- A field holding text in some documents and objects or arrays in others takes no value that reads as JSON from the grid, since it could be either type: the save is refused. Write it with a query. +- A new row, duplicate or paste cannot hold a field named `__proto__`, or an empty field name at any depth: the save is refused. Set that cell to NULL, save, then set it on the saved document. - `_id` is read-only in the grid, and a row added with **Add Row** is inserted without one so the server generates it. To choose your own, use **Insert Document…**. - Transactions are not exposed. Statements always run standalone, on any topology. - A collection takes one text index. A second **FULLTEXT** row fails after the collection and the indexes before it are created: list every text field in one index instead. diff --git a/project.yml b/project.yml index 230a3d1fba..49281641d2 100644 --- a/project.yml +++ b/project.yml @@ -512,6 +512,7 @@ targets: - Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift - Plugins/MongoDBDriverPlugin/MongoDBCollectionSchema.swift - Plugins/MongoDBDriverPlugin/MongoDBCreateDatabasePlan.swift + - Plugins/MongoDBDriverPlugin/MongoDBFieldKinds.swift - Plugins/MongoDBDriverPlugin/MongoDBFilterClause.swift - Plugins/MongoDBDriverPlugin/MongoDBFilterValue.swift - Plugins/MongoDBDriverPlugin/MongoDBFindLimitPolicy.swift From 7657818ce13d028b470cf4be7fba96732d57bc34 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 27 Sep 2026 01:55:02 +0700 Subject: [PATCH 3/3] fix(plugin-mongodb): keep JSON-shaped text a string in string fields, read wrappers by their whole shape, and refuse a pasted nested leaf its array does not hold --- .../ElasticsearchStatementGenerator.swift | 36 +++++++++++++++---- .../MongoDBStatementGenerator.swift | 21 +++++++++-- .../MongoExtendedJsonForm.swift | 19 ++++++++-- .../Plugins/ElasticsearchDriverTests.swift | 26 ++++++++++++++ .../MongoDBNestedValueWriteTests.swift | 33 +++++++++++++++++ 5 files changed, 123 insertions(+), 12 deletions(-) diff --git a/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift b/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift index b4c46d5561..4d76453f0b 100644 --- a/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift +++ b/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift @@ -122,8 +122,9 @@ struct ElasticsearchStatementGenerator { return .init(method: "POST", path: "/\(encodedIndex)/_doc\(Self.refreshQuery)", body: body) } - /// A new row's leaf value reaches the server only inside its array, so one the user typed, or - /// one whose array is empty, would be dropped. Metadata other than `_id` is the server's to set. + /// A new row's leaf value reaches the server only inside its array, so one the user typed, one + /// whose array is empty, or one that says something other than its array would be dropped. + /// Metadata other than `_id` is the server's to set. private func unwritableInsertValue(in change: PluginRowChange, values: [String: PluginCellValue]) -> String? { for cellChange in change.cellChanges where !cellChange.newValue.isNull { let column = cellChange.columnName @@ -135,15 +136,36 @@ struct ElasticsearchStatementGenerator { } } for column in columns { - guard let parent = nestedParentByLeaf[column], - values[column]?.isNull == false, - values[parent]?.isNull ?? true - else { continue } - return Self.nestedLeafReason(leaf: column, parent: parent) + guard let parent = nestedParentByLeaf[column], let leaf = values[column], !leaf.isNull else { continue } + guard let array = values[parent]?.asText, + Self.sameCell(leaf, projectedLeaf(column, of: parent, arrayText: array)) else { + return Self.nestedLeafReason(leaf: column, parent: parent) + } } return nil } + /// What the grid shows for `leaf` once `arrayText` is saved into `parent`, read by the same + /// flattener that reads the document back. + private func projectedLeaf(_ leaf: String, of parent: String, arrayText: String) -> PluginCellValue { + let placed = parent.split(separator: ".").reversed().reduce(jsonValue(arrayText, for: parent)) { inner, key in + [String(key): inner] + } + guard let source = placed as? [String: Any] else { return .null } + return ElasticsearchMappingFlattener.flattenSource(source, nestedParents: [parent])[leaf] ?? .null + } + + /// Two cells that hold the same JSON value compare equal however it is spaced. + private static func sameCell(_ lhs: PluginCellValue, _ rhs: PluginCellValue) -> Bool { + guard let left = lhs.asText, let right = rhs.asText else { return lhs.isNull && rhs.isNull } + guard let leftValue = parsedJSON(left), let rightValue = parsedJSON(right) else { return left == right } + return (leftValue as AnyObject).isEqual(rightValue) + } + + private static func parsedJSON(_ text: String) -> Any? { + try? JSONSerialization.jsonObject(with: Data(text.utf8), options: .fragmentsAllowed) + } + // MARK: - UPDATE private func updateRequest(for change: PluginRowChange) throws -> ElasticsearchWriteRequest? { diff --git a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index 72d020671b..319b6519b5 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -541,6 +541,8 @@ struct MongoDBStatementGenerator { return try shellJson(parsed, field: field) case .jsonOrString: return try parsed.map { try shellJson($0, field: field) } + case .text: + return nil case .ambiguous: throw MongoDBWriteRefusal.documentOrText(field: field) } @@ -554,6 +556,8 @@ struct MongoDBStatementGenerator { case container /// JSON is a document or an array, and any other text a string. case jsonOrString + /// A string, whatever it reads like. + case text /// The field holds both, and nothing says which this value is meant to be. case ambiguous } @@ -562,7 +566,9 @@ struct MongoDBStatementGenerator { /// JSON is a document or an array where the field held those and no strings, and could be either /// where it held both, which is refused. Text that is not JSON can only be a string when it was /// read from a cell, where a document always shows as JSON, or typed over a value that was not - /// one. An existing value whose field was never read says nothing either way. + /// one. An existing value whose field was never read says nothing either way. In a field that + /// held strings and never a document or an array, a value put back, copied, or typed over a + /// string is a string too, however it reads: `{"a":1}` stored as text stays text. private func containerReading(of field: String, isJSON: Bool, provenance: Provenance) -> ContainerReading { if let replaced = provenance.replaced, holdsContainer(replaced, field: field) { return .container @@ -570,7 +576,9 @@ struct MongoDBStatementGenerator { guard let held = heldKinds(of: field) else { return isJSON && standsForUnreadValue(provenance) ? .ambiguous : .jsonOrString } - guard held.contains(.document) || held.contains(.array) else { return .jsonOrString } + guard held.contains(.document) || held.contains(.array) else { + return held.contains(.string) && continuesAString(provenance) ? .text : .jsonOrString + } guard held.contains(.string) else { return .container } return !isJSON && nonJSONCanOnlyBeText(provenance) ? .jsonOrString : .ambiguous } @@ -583,6 +591,15 @@ struct MongoDBStatementGenerator { return held.contains(shape) && !held.contains(.string) } + private func continuesAString(_ provenance: Provenance) -> Bool { + switch provenance { + case .restored, .copiedIntoNewDocument, .edit(replacing: .text): + return true + case .edit, .typedIntoNewDocument: + return false + } + } + private func nonJSONCanOnlyBeText(_ provenance: Provenance) -> Bool { switch provenance { case .restored, .copiedIntoNewDocument: diff --git a/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift b/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift index 9003bb0307..d3266b3719 100644 --- a/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift +++ b/Plugins/MongoDBDriverPlugin/MongoExtendedJsonForm.swift @@ -26,10 +26,23 @@ enum MongoExtendedJsonForm { "$undefined", "$uuid", "$regex" ] - /// An object that stands for one BSON value rather than an embedded document. + /// The keys a wrapper may hold beside the one that opens it, in the legacy forms libbson reads. + private static let companionKeys: [String: Set] = [ + "$binary": ["$type"], "$code": ["$scope"], "$regex": ["$options"] + ] + + /// An object that stands for one BSON value rather than an embedded document: it opens with a + /// wrapper key and holds nothing that wrapper does not take. An object that opens with `$oid` + /// and goes on to other members is a document, so its other members are written and checked + /// like any document's rather than dropped with the wrapper. static func isWrapper(_ members: [Member]) -> Bool { - guard let first = members.first else { return false } - return wrapperKeys.contains(first.key) + guard let first = members.first, wrapperKeys.contains(first.key) else { return false } + let allowed = companionKeys[first.key] ?? [] + var seen: Set = [first.key] + for member in members.dropFirst() { + guard allowed.contains(member.key), seen.insert(member.key).inserted else { return false } + } + return true } // MARK: - Display diff --git a/TableProTests/Plugins/ElasticsearchDriverTests.swift b/TableProTests/Plugins/ElasticsearchDriverTests.swift index ca73b766e7..ffdd8d8c43 100644 --- a/TableProTests/Plugins/ElasticsearchDriverTests.swift +++ b/TableProTests/Plugins/ElasticsearchDriverTests.swift @@ -949,6 +949,32 @@ struct ElasticsearchStatementGeneratorTests { } } + @Test("A pasted row whose leaf says something other than its array is refused, not saved without it") + func insertRefusesPastedLeafThatDisagreesWithItsArray() { + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + #expect(throws: nestedLeafRefusal()) { + try nestedGenerator().generateRowWrites( + from: [change], + insertedRowData: [0: [.null, .text("[{\"type\":\"CPF\"}]"), .text("[\"CNPJ\"]"), .text("p1")]], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + } + } + + @Test("A pasted leaf that holds what its array holds, however spaced, is written through the array") + func insertAcceptsPastedLeafThatMatchesItsArray() throws { + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let statements = try nestedGenerator().generateRowWrites( + from: [change], + insertedRowData: [0: [.null, .text("[{\"type\":\"CPF\"},{\"type\":\"RG\"}]"), .text("[ \"CPF\", \"RG\" ]"), .null]], + deletedRowIndices: [], + insertedRowIndices: [0] + ) + let body = ElasticsearchStatementGenerator.decode(statements[0].statement)?.body + #expect(body?.contains("\"identifiers\":[{\"type\":\"CPF\"},{\"type\":\"RG\"}]") == true) + } + @Test("A pasted row carrying a leaf value with no array to write it through is refused") func insertRefusesPastedLeafWithoutItsArray() { let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) diff --git a/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift b/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift index 85d401e756..8f41d4353b 100644 --- a/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift +++ b/TableProTests/Plugins/MongoDBNestedValueWriteTests.swift @@ -278,6 +278,22 @@ struct MongoDBNestedValueWriteTests { #expect(copied == #"db.people.insertOne({"f": "{abc"})"#) } + @Test("Text that reads as JSON in a field that only held strings stays a string when copied, restored or typed over") + func stringOnlyFieldKeepsJSONShapedText() throws { + let copied = try insert("f", #"{"a":1}"#, held: [.string]) + let edited = try update("f", from: .text("plain"), to: "[1,2]", held: [.string], majority: .string) + let typed = try insert("f", #"{"a":1}"#, held: [.string], typed: true) + let strings = MongoDBStatementGenerator( + collectionName: "people", columns: ["_id", "f"], fieldKinds: MongoDBFieldKinds(["f": [.string]]) + ) + let restored = try #require(strings.generateRestore(rows: [["507f1f77bcf86cd799439011", #"{"a":1}"#]])?.first) + + #expect(copied == #"db.people.insertOne({"f": "{\"a\":1}"})"#) + #expect(edited == #"db.people.updateOne({"_id": 1}, {"$set": {"f": "[1,2]"}})"#) + #expect(typed.contains(#""f": {"a":1}"#)) + #expect(restored.statement.contains(#""f": "{\"a\":1}""#)) + } + @Test("A restore that reads as JSON in a field never read is refused, and one that does not is a string") func restoreIntoAnUnreadField() throws { let relaunched = MongoDBStatementGenerator(collectionName: "people", columns: ["_id", "f"]) @@ -358,6 +374,23 @@ struct MongoDBNestedValueWriteTests { #expect(ordered.contains(#""meta": {"2":"two","10":"ten","zip":"1"}"#)) } + @Test("An object that opens with a wrapper key and holds other members is a document, so its keys are checked") + func wrapperShapedDocumentIsCheckedAsADocument() throws { + typealias Member = MongoDocumentText.Member + let hex = MongoDocumentText.Value.string("507f1f77bcf86cd799439011") + + #expect(throws: MongoDBWriteRefusal.reorderedByTheShell(field: "meta").refusal(ofRow: 0)) { + try insert("meta", #"{"$oid":"x","__proto__":1}"#) + } + #expect(MongoExtendedJsonForm.isWrapper([Member(key: "$oid", value: hex)])) + #expect(!MongoExtendedJsonForm.isWrapper([Member(key: "$oid", value: hex), Member(key: "note", value: hex)])) + #expect(MongoExtendedJsonForm.isWrapper([Member(key: "$binary", value: hex), Member(key: "$type", value: hex)])) + #expect(MongoExtendedJsonForm.isWrapper([Member(key: "$regex", value: hex), Member(key: "$options", value: hex)])) + #expect(!MongoExtendedJsonForm.isWrapper([ + Member(key: "$regex", value: hex), Member(key: "$options", value: hex), Member(key: "$options", value: hex) + ])) + } + @Test("Only array-index keys up to 4294967294 are moved, so a larger number-like key is written where it is") func shellKeyOrderStopsAtTheArrayIndexRange() throws { #expect(throws: MongoDBWriteRefusal.reorderedByTheShell(field: "meta").refusal(ofRow: 0)) {