fix(plugin-mongodb): store Set NULL as null, tell a missing field from null, and add Remove Field

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - **Insert Document…** for MongoDB collections, written as Extended JSON. (#3132)
+- **Remove Field** for MongoDB cells, and **No Field** for a field a document does not have. (#3132)
 - **Agent** mode: one session with the whole connection window, sessions to start and delete, and what each one ran.
 - Row previews and the query editor sized to the display on iPad and on iPhone Duo's inner display.
 - Table list and table browser side by side on iPad and on iPhone Duo's inner display.
@@ -139,6 +140,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Set DEFAULT** on a MongoDB field storing the text `__DEFAULT__`. (#3132)
 - Edits to MongoDB fields named with a dot, a leading `$` or `__proto__` changing another field or nothing. (#3132)
 - Nested MongoDB values shown with sorted keys, and saved with ObjectIds, dates and numbers retyped. (#3132)
+- **Set NULL** on a MongoDB field deleting the field. (#3132)
+- Null fields dropped from duplicated, pasted or restored MongoDB documents. (#3132)
+- **Restore Previous Values** comparing the wrong MongoDB fields when a document lacks one. (#3132)
+- Row inspector accepting edits to a MongoDB `_id` or another read-only field that were never saved. (#3132)
 - `tablepro-mcp` crashing when its standard input was non-blocking.
 - `tablepro-mcp` using a full CPU core, or crashing, when its standard output or error was non-blocking.
 - Server connections piling up while browsing many databases or schemas, and staying open after a failed connect. (#3103)

Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift

@@ -98,6 +98,17 @@ struct BsonDocumentFlattener {
         }
     }
 
+    /// The columns each document has no field for, by row. A field holding null is there, and only
+    /// rows missing something appear.
+    static func absentCells(of documents: [[String: Any]], columns: [String]) -> [Int: Set<Int>] {
+        var absent: [Int: Set<Int>] = [:]
+        for (row, document) in documents.enumerated() {
+            let missing = Set(columns.indices.filter { document[columns[$0]] == nil })
+            if !missing.isEmpty { absent[row] = missing }
+        }
+        return absent
+    }
+
     /// A document or an array, and not the `$code` or DBRef shapes that render as their own text.
     private static func isNestedValue(_ value: Any) -> Bool {
         if value is [Any] { return true }

Plugins/MongoDBDriverPlugin/MongoDBCollectionSchema.swift

@@ -48,6 +48,10 @@ struct MongoDBDeclaredField: Equatable, Sendable {
     let bsonTypes: [String]
     let isRequired: Bool
     let allowedValues: [String]?
+    /// Whether the validator lets the field hold null. Separate from `isRequired`, which is about
+    /// the field being there at all: a required field typed `["date", "null"]` takes null, and an
+    /// optional field typed `string` refuses it.
+    var admitsNull = true
 
     var valueKind: BsonValueKind? {
         let valueTypes = bsonTypes.filter { $0 != "null" }
@@ -83,6 +87,16 @@ enum MongoDBCollectionShape {
 
 struct MongoDBCollectionSchema: Equatable, Sendable {
     let fields: [MongoDBDeclaredField]
+    /// Whether a field the validator's `properties` do not name may hold null: an
+    /// `additionalProperties` schema decides it for every such field.
+    var undeclaredFieldsAdmitNull = true
+    /// A rule this reader does not model reaches every field: a combinator, `patternProperties`,
+    /// `dependencies` or an `enum` over the whole document, or a query operator beside
+    /// `$jsonSchema`. Each can refuse null in any field, so none is taken to admit it.
+    var hasUnmodeledRule = false
+    /// False for `validationAction: "warn"` and `validationLevel: "off"`, where the server stores
+    /// what the validator would refuse.
+    var isEnforced = true
 
     static let empty = MongoDBCollectionSchema(fields: [])
 
@@ -92,6 +106,13 @@ struct MongoDBCollectionSchema: Equatable, Sendable {
         fields.first { $0.name == name }
     }
 
+    /// Whether the server takes null in this field, which is what offers **Set Value > NULL**.
+    func admitsNull(fieldNamed name: String) -> Bool {
+        guard isEnforced else { return true }
+        guard !hasUnmodeledRule else { return false }
+        return field(named: name)?.admitsNull ?? undeclaredFieldsAdmitNull
+    }
+
     var valueKinds: [String: BsonValueKind] {
         var kinds: [String: BsonValueKind] = [:]
         for field in fields {
@@ -127,32 +148,44 @@ struct MongoDBCollectionSchema: Equatable, Sendable {
               let batch = MongoScriptJson.member(of: cursor, key: "firstBatch"),
               let collection = MongoScriptJson.topLevelElements(batch).first,
               let options = MongoScriptJson.member(of: collection, key: "options"),
-              let validator = MongoScriptJson.member(of: options, key: "validator"),
-              let jsonSchema = MongoScriptJson.member(of: validator, key: "$jsonSchema") else {
+              let validator = MongoScriptJson.member(of: options, key: "validator") else {
             return .empty
         }
-        return parse(jsonSchema: jsonSchema)
+        var schema = MongoScriptJson.member(of: validator, key: "$jsonSchema").map { parse(jsonSchema: $0) } ?? .empty
+        if MongoScriptJson.members(of: validator).contains(where: { $0.key != "$jsonSchema" }) {
+            schema.hasUnmodeledRule = true
+        }
+        let settings = decodeObject(options) ?? [:]
+        schema.isEnforced = settings["validationAction"] as? String != "warn"
+            && settings["validationLevel"] as? String != "off"
+        return schema
     }
 
     static func parse(jsonSchema: String) -> MongoDBCollectionSchema {
-        guard let schema = decodeObject(jsonSchema),
-              let properties = schema["properties"] as? [String: Any],
-              let propertiesText = MongoScriptJson.member(of: jsonSchema, key: "properties") else {
-            return .empty
+        guard let schema = decodeObject(jsonSchema) else { return .empty }
+        return MongoDBCollectionSchema(
+            fields: declaredFields(in: schema, text: jsonSchema),
+            undeclaredFieldsAdmitNull: undeclaredFieldsAdmitNull(schema["additionalProperties"]),
+            hasUnmodeledRule: documentWideRules.contains { schema[$0] != nil }
+        )
+    }
+
+    private static func declaredFields(in schema: [String: Any], text: String) -> [MongoDBDeclaredField] {
+        guard let properties = schema["properties"] as? [String: Any],
+              let propertiesText = MongoScriptJson.member(of: text, key: "properties") else {
+            return []
         }
         let required = Set(schema["required"] as? [String] ?? [])
-        let orderedNames = orderedKeys(of: propertiesText, in: properties)
-
-        let fields = orderedNames.compactMap { name -> MongoDBDeclaredField? in
+        return orderedKeys(of: propertiesText, in: properties).compactMap { name -> MongoDBDeclaredField? in
             guard let spec = properties[name] as? [String: Any] else { return nil }
             return MongoDBDeclaredField(
                 name: name,
                 bsonTypes: declaredTypes(in: spec),
                 isRequired: required.contains(name),
-                allowedValues: stringEnum(spec["enum"])
+                allowedValues: stringEnum(spec["enum"]),
+                admitsNull: admitsNull(spec)
             )
         }
-        return MongoDBCollectionSchema(fields: fields)
     }
 
     private static func orderedKeys(of objectText: String, in decoded: [String: Any]) -> [String] {
@@ -174,6 +207,25 @@ struct MongoDBCollectionSchema: Equatable, Sendable {
         return []
     }
 
+    /// The server applies every keyword of a field's rule, so null passes only when each one that
+    /// can refuse it lets it through: a declared type has to list null, and an `enum` has to hold
+    /// it. A combinator is not modeled and is taken to refuse null. The keywords for strings,
+    /// numbers, arrays and documents never see null, measured on 7.0 for `minLength` and `pattern`.
+    private static func admitsNull(_ spec: [String: Any]) -> Bool {
+        let typeAdmitsNull = (spec["bsonType"] == nil && spec["type"] == nil) || declaredTypes(in: spec).contains("null")
+        let enumAdmitsNull = (spec["enum"] as? [Any])?.contains { $0 is NSNull } ?? true
+        return typeAdmitsNull && enumAdmitsNull && !unmodeledConstraints.contains { spec[$0] != nil }
+    }
+
+    private static let unmodeledConstraints = ["anyOf", "oneOf", "allOf", "not"]
+
+    private static let documentWideRules = unmodeledConstraints + ["dependencies", "patternProperties", "enum"]
+
+    private static func undeclaredFieldsAdmitNull(_ additionalProperties: Any?) -> Bool {
+        if let spec = additionalProperties as? [String: Any] { return admitsNull(spec) }
+        return additionalProperties as? Bool ?? true
+    }
+
     private static func stringEnum(_ value: Any?) -> [String]? {
         guard let array = value as? [Any], !array.isEmpty,
               array.allSatisfy({ $0 is String }) else { return nil }

Plugins/MongoDBDriverPlugin/MongoDBPlugin.swift

@@ -16,6 +16,7 @@ final class MongoDBPlugin: NSObject, TableProPlugin, DriverPlugin {
 
     static let supportsRenameTable = true
     static let supportsDocumentEditing = true
+    static let supportsFieldRemoval = true
     static let databaseDisplayName = "MongoDB"
     static let iconName = "mongodb-icon"
     static let defaultPort = 27_017

Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift

@@ -266,7 +266,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
         guard let rowCap, MongoDBFindLimitPolicy.isTruncated(rowCount: result.rows.count, rowCap: rowCap) else {
             return result
         }
-        return PluginQueryResult(
+        var capped = PluginQueryResult(
             columns: result.columns,
             columnTypeNames: result.columnTypeNames,
             rows: Array(result.rows.prefix(rowCap)),
@@ -275,6 +275,8 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
             isTruncated: true,
             statusMessage: result.statusMessage
         )
+        capped.absentCells = result.absentCells?.filter { $0.key < rowCap }
+        return capped
     }
 
     private func mapExecutionError(_ error: Error) -> Error {
@@ -348,7 +350,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
             PluginColumnInfo(
                 name: name,
                 dataType: BsonDocumentFlattener.typeName(for: kinds[index], representation: uuidRepresentation),
-                isNullable: name != MongoDBCollectionDDL.idField && schema.field(named: name)?.isRequired != true,
+                isNullable: name != MongoDBCollectionDDL.idField && schema.admitsNull(fieldNamed: name),
                 isPrimaryKey: name == MongoDBCollectionDDL.idField,
                 defaultValue: nil, extra: nil, charset: nil, collation: nil, comment: nil,
                 allowedValues: schema.allowedValues[name]
@@ -365,7 +367,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
             name: name,
             dataType: field?.columnTypeName(representation: uuidRepresentation)
                 ?? BsonDocumentFlattener.typeName(for: .objectId, representation: uuidRepresentation),
-            isNullable: !isKey && field?.isRequired != true,
+            isNullable: !isKey && schema.admitsNull(fieldNamed: name),
             isPrimaryKey: isKey,
             defaultValue: nil, extra: nil, charset: nil, collation: nil, comment: nil,
             allowedValues: schema.allowedValues[name]
@@ -807,9 +809,10 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
         schema: String?,
         columns: [String],
         primaryKeyColumns: [String],
-        rows: [[PluginCellValue]]
+        rows: [[PluginCellValue]],
+        absentCells: [Int: Set<Int>]
     ) -> [(statement: String, parameters: [PluginCellValue])]? {
-        writeGenerator(for: table, columns: columns).generateRestore(rows: rows)
+        writeGenerator(for: table, columns: columns).generateRestore(rows: rows, absentCells: absentCells)
     }
 
     private func writeGenerator(for table: String, columns: [String]) -> MongoDBStatementGenerator {
@@ -955,12 +958,14 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable {
             representation: uuidRepresentation, storedTexts: read.texts
         )
 
-        return PluginQueryResult(
+        var result = PluginQueryResult(
             columns: columns, columnTypeNames: typeNames,
             rows: rows, rowsAffected: 0,
             executionTime: Date().timeIntervalSince(startTime),
             isTruncated: isTruncated
         )
+        result.absentCells = BsonDocumentFlattener.absentCells(of: documents, columns: columns)
+        return result
     }
 
     // MARK: - Helpers

Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift

@@ -110,32 +110,38 @@ struct MongoDBStatementGenerator {
 
     // MARK: - INSERT
 
-    /// NULL and DEFAULT leave a field out of a new document. A row with nothing else in it is the
-    /// empty document, which the server stores with a generated `_id`.
+    /// A field the new row has none of, and DEFAULT, leave the field out of the document; NULL is
+    /// written as null, except in `_id`, which the grid never lets anyone set. A row with nothing
+    /// else in it is the empty document, which the server stores with a generated `_id`.
     ///
     /// A new row's change lists the cells the user filled in; every other value came with the row,
     /// copied from the row it duplicates or pastes.
     private func insertStatement(
         for change: PluginRowChange,
         insertedRowData: [Int: [PluginCellValue]]
     ) throws -> String {
-        let filledIn = change.cellChanges.map { (field: $0.columnName, value: $0.newValue) }
+        let filledIn = change.cellChanges.map { (column: $0.columnIndex, field: $0.columnName, value: $0.newValue) }
         let typed = Dictionary(filledIn.map { ($0.field, $0.value) }, uniquingKeysWith: { _, last in last })
-        let cells: [(field: String, value: PluginCellValue)]
+        let cells: [(column: Int, field: String, value: PluginCellValue)]
         if let values = insertedRowData[change.rowIndex] {
-            cells = zip(columns, values).map { (field: $0, value: $1) }
+            cells = zip(columns, values).enumerated().map { (column: $0, field: $1.0, value: $1.1) }
         } else {
             cells = filledIn
         }
-        let entries = try cells.filter { !isLeftOut($0.value) }.map { cell in
+        let absent = change.absentColumns ?? []
+        let written = cells.filter { cell in
+            let serverAssignsIdentity = cell.field == MongoDBCollectionDDL.idField && cell.value.isNull
+            return !serverAssignsIdentity && !isLeftOut(cell.value, isAbsent: absent.contains(cell.column))
+        }
+        let entries = try written.map { cell in
             let provenance: Provenance = typed[cell.field] == cell.value ? .typedIntoNewDocument : .copiedIntoNewDocument
             return "\(quotedKey(cell.field)): \(try documentValueJson(cell.value, field: cell.field, provenance: provenance))"
         }
         return "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})"
     }
 
-    private func isLeftOut(_ value: PluginCellValue) -> Bool {
-        value.isNull || value.asText == Self.defaultMarker
+    private func isLeftOut(_ value: PluginCellValue, isAbsent: Bool) -> Bool {
+        isAbsent || value.asText == Self.defaultMarker
     }
 
     /// A value of a whole document the grid writes: a new row, a duplicate or a paste, or a
@@ -164,16 +170,21 @@ struct MongoDBStatementGenerator {
     /// A new row leaves `_id` to the server. Undoing a delete is the opposite requirement: a new
     /// `_id` is a different document, and anything that referenced the old one still points at
     /// nothing. A value that cannot be written refuses the restore rather than dropping the field.
-    func generateRestore(rows: [[PluginCellValue]]) -> [(statement: String, parameters: [PluginCellValue])]? {
+    /// A field the document did not have stays missing, and one that held null holds it again.
+    func generateRestore(
+        rows: [[PluginCellValue]],
+        absentCells: [Int: Set<Int>] = [:]
+    ) -> [(statement: String, parameters: [PluginCellValue])]? {
         guard let idIndex = idColumnIndex else { return nil }
 
         do {
-            return try rows.map { row in
+            return try rows.enumerated().map { rowIndex, row in
                 guard idIndex < row.count else { throw MongoDBWriteRefusal.missingIdentity }
+                let absent = absentCells[rowIndex] ?? []
                 let idField = MongoDBCollectionDDL.idField
                 var entries = ["\(quotedKey(idField)): \(try documentValueJson(row[idIndex], field: idField, provenance: .restored))"]
                 for (index, value) in row.enumerated() where index != idIndex && index < columns.count {
-                    guard !isLeftOut(value) else { continue }
+                    guard !isLeftOut(value, isAbsent: absent.contains(index)) else { continue }
                     let field = columns[index]
                     entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field, provenance: .restored))")
                 }
@@ -204,21 +215,32 @@ struct MongoDBStatementGenerator {
         }
     }
 
+    /// A field the change names in `absentColumns` is removed with `$unset`; NULL is stored as null.
     private func updateStatement(for change: PluginRowChange) throws -> String? {
         guard !change.cellChanges.isEmpty else { return nil }
         let identity = try identityJson(of: change)
+        let removed = change.absentColumns ?? []
         let cellWrites = try change.cellChanges.map {
-            try cellWrite(field: $0.columnName, from: $0.oldValue, to: $0.newValue)
+            try cellWrite(
+                field: $0.columnName, from: $0.oldValue, to: $0.newValue,
+                removesField: removed.contains($0.columnIndex)
+            )
         }
         let update = try updateDocument(for: cellWrites)
         return "\(collectionAccessor).updateOne({\"_id\": \(identity)}, \(update))"
     }
 
-    private func cellWrite(field: String, from oldValue: PluginCellValue, to newValue: PluginCellValue) throws -> CellWrite {
+    private func cellWrite(
+        field: String,
+        from oldValue: PluginCellValue,
+        to newValue: PluginCellValue,
+        removesField: Bool
+    ) throws -> CellWrite {
         guard field != MongoDBCollectionDDL.idField else { throw MongoDBWriteRefusal.identityChanged }
+        guard !removesField else { return .remove(field: field) }
         switch newValue {
         case .null:
-            return .remove(field: field)
+            return .whole(field: field, json: "null")
         case .bytes(let data):
             return .whole(field: field, json: try binaryJson(data, field: field, provenance: .edit(replacing: oldValue)))
         case .text(let text):