fix(plugin-mongodb): send $type, $regex and $options objects as the operator documents they are written as

.github/workflows/macos-tests.yml

@@ -183,9 +183,9 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: scripts/download-libs.sh
 
-      # Compiles a C probe against Libs/libbson and parses every filter document the MongoDB
-      # query builder can emit. It guarded that invariant and ran nowhere; it needs clang and the
-      # vendored libraries, so this is the first job where it can run at all.
+      # Compiles the MongoDB driver's own BSON planner and builder against Libs/libbson and builds
+      # every filter document the query builder can emit, checking the BSON type where it matters.
+      # It needs swiftc and the vendored libraries, so this is the first job where it can run.
       - name: Check the MongoDB filter shapes against the query builder
         run: scripts/check-mongodb-filter-shapes.sh
 

CHANGELOG.md

@@ -117,6 +117,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - ⌘F finds and ⇧⌘F toggles filters in data file windows, as in the table grid.
 - Large data files opened, filtered, sorted and searched in the background, with progress and Cancel.
 - `.json` and `.ndjson` files opened in the Data Files window rather than as a DuckDB connection.
+- `$regex` and `$options` objects in MongoDB scripts and **Raw Filter** sent as operator documents, as in mongosh.
 
 ### Removed
 
@@ -529,6 +530,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - MongoDB collections could not be created from **New Table…**. (#3131)
 - A new or empty MongoDB collection showing only `_id` instead of the fields its validator declares.
 - MongoDB edits that stored dates and ObjectIds as text, rounded integers past 2^53, or missed a string `_id` that looks numeric.
+- MongoDB filters, validators and pipelines with a `$type` or `$regex` object refused as "not a document MongoDB can read".
 - **New Table…** offered on databases that cannot create a table, such as Redis and Kafka.
 - Executing indicator and Stop button carried over for a moment onto the query tab switched to.
 

Plugins/MongoDBDriverPlugin/MongoBsonAssembly.swift

@@ -0,0 +1,146 @@
+import Foundation
+
+/// How one JSON text becomes one BSON document when libbson's reader would misread part of it.
+///
+/// libbson decides what an embedded object is from its first key alone. `$type`, `$regex` and
+/// `$options` open its legacy binary and regular expression values, and they are also query
+/// operators: `{"sig": {"$type": "binData"}}` fails with `Missing "$binary"`,
+/// `{"$regex": "^a", "$exists": true}` fails with `Invalid key "$exists"`, and
+/// `{"$regex": "a", "$options": "i"}` becomes a regular expression value. mongosh sends each of them
+/// as the document it is written as. So does this: such an object is built member by member, and a
+/// member is handed to libbson as a document of its own, `{"key": value}`, where a key is never
+/// special. Every value that holds no such object is still read by libbson as written, wrappers
+/// such as `{"$oid": …}` included.
+enum MongoBsonAssembly: Equatable, Sendable {
+    /// Text libbson reads as meant, which is nearly all of it.
+    case whole(String)
+    /// A document built from these parts, in order.
+    case parts([Part])
+
+    enum Part: Equatable, Sendable {
+        /// A document of one member, which libbson reads as written.
+        case member(String)
+        case document(key: String, parts: [Part])
+        /// Its parts are keyed `0`, `1` and on, the keys of a BSON array.
+        case array(key: String, parts: [Part])
+    }
+
+    /// libbson's special keys that are also MongoDB operators.
+    static let operatorKeys: Set<String> = ["$type", "$regex", "$options"]
+
+    /// libbson's other special keys, each opening an Extended JSON value that is never taken apart.
+    static let valueKeys: Set<String> = [
+        "$binary", "$code", "$date", "$dbPointer", "$maxKey", "$minKey", "$numberDecimal", "$numberDouble",
+        "$numberInt", "$numberLong", "$oid", "$regularExpression", "$scope", "$symbol", "$timestamp",
+        "$undefined", "$uuid"
+    ]
+
+    static func plan(_ json: String) -> MongoBsonAssembly {
+        guard holdsOperatorDocument(json), isWellFormed(json) else { return .whole(json) }
+        let trimmed = json.trimmingCharacters(in: .whitespacesAndNewlines)
+        if trimmed.hasPrefix("[") { return .parts(elementParts(of: trimmed)) }
+        return .parts(memberParts(of: trimmed))
+    }
+
+    // MARK: - Planning
+
+    private static func memberParts(of objectJson: String) -> [Part] {
+        MongoScriptJson.members(of: objectJson).map { part(key: $0.key, value: $0.value) }
+    }
+
+    private static func elementParts(of arrayJson: String) -> [Part] {
+        MongoScriptJson.topLevelElements(arrayJson).enumerated().map { part(key: String($0.offset), value: $0.element) }
+    }
+
+    private static func part(key: String, value: String) -> Part {
+        guard isTakenApart(value) else { return .member("{\(MongoScriptJson.jsonString(key)):\(value)}") }
+        if value.hasPrefix("[") { return .array(key: key, parts: elementParts(of: value)) }
+        return .document(key: key, parts: memberParts(of: value))
+    }
+
+    /// An operator document is taken apart, and so is anything holding one. A value wrapper is
+    /// not, whatever it holds: taking a `$code` with a `$scope` apart would store a document where
+    /// the script wrote code.
+    private static func isTakenApart(_ valueJson: String) -> Bool {
+        guard valueJson.hasPrefix("{") || valueJson.hasPrefix("["),
+              holdsOperatorDocument(valueJson, countingOutermost: true) else { return false }
+        guard valueJson.hasPrefix("{") else { return true }
+        let firstKey = MongoScriptJson.members(of: valueJson).first?.key
+        return firstKey.map { !valueKeys.contains($0) } ?? false
+    }
+
+    private static func isWellFormed(_ json: String) -> Bool {
+        (try? JSONSerialization.jsonObject(with: Data(json.utf8), options: .fragmentsAllowed)) != nil
+    }
+
+    // MARK: - Scanning
+
+    /// Whether an object opens with an operator key once its escapes are decoded, as libbson
+    /// decodes them. libbson never misreads the outermost object of what it parses, so that one
+    /// counts only when `countingOutermost` is set.
+    static func holdsOperatorDocument(_ json: String, countingOutermost: Bool = false) -> Bool {
+        var text = json
+        text.makeContiguousUTF8()
+        let shallowest = countingOutermost ? 1 : 2
+        return text.utf8.withContiguousStorageIfAvailable { scan($0, fromDepth: shallowest) } ?? false
+    }
+
+    private static let quote = UInt8(ascii: "\"")
+    private static let backslash = UInt8(ascii: "\\")
+    private static let dollar = UInt8(ascii: "$")
+
+    private static func scan(_ bytes: UnsafeBufferPointer<UInt8>, fromDepth shallowest: Int) -> Bool {
+        var depth = 0
+        var awaitsFirstKey = false
+        var index = 0
+
+        while index < bytes.count {
+            let byte = bytes[index]
+            switch byte {
+            case quote:
+                let end = stringEnd(in: bytes, from: index + 1)
+                if awaitsFirstKey, depth >= shallowest, isOperatorKey(bytes[(index + 1) ..< end]) { return true }
+                awaitsFirstKey = false
+                index = end
+            case UInt8(ascii: "{"):
+                depth += 1
+                awaitsFirstKey = true
+            case UInt8(ascii: "["):
+                depth += 1
+                awaitsFirstKey = false
+            case UInt8(ascii: "}"), UInt8(ascii: "]"):
+                depth -= 1
+                awaitsFirstKey = false
+            case UInt8(ascii: " "), UInt8(ascii: "\t"), UInt8(ascii: "\n"), UInt8(ascii: "\r"):
+                break
+            default:
+                awaitsFirstKey = false
+            }
+            index += 1
+        }
+        return false
+    }
+
+    /// The index of the quote that closes the string whose contents start at `start`.
+    private static func stringEnd(in bytes: UnsafeBufferPointer<UInt8>, from start: Int) -> Int {
+        var index = start
+        while index < bytes.count {
+            switch bytes[index] {
+            case backslash: index += 2
+            case quote: return index
+            default: index += 1
+            }
+        }
+        return bytes.count
+    }
+
+    private static let operatorKeyBytes = operatorKeys.map { Array($0.utf8) }
+
+    private static func isOperatorKey(_ raw: Slice<UnsafeBufferPointer<UInt8>>) -> Bool {
+        guard let first = raw.first, first == dollar || first == backslash else { return false }
+        guard raw.contains(backslash) else { return operatorKeyBytes.contains { $0.elementsEqual(raw) } }
+        let quoted = Data([quote] + raw + [quote])
+        let decoded = try? JSONSerialization.jsonObject(with: quoted, options: .fragmentsAllowed) as? String
+        return decoded.map(operatorKeys.contains) ?? false
+    }
+}

Plugins/MongoDBDriverPlugin/MongoBsonBuilder.swift

@@ -0,0 +1,77 @@
+//
+//  MongoBsonBuilder.swift
+//  MongoDBDriverPlugin
+//
+
+#if canImport(CLibMongoc)
+import CLibMongoc
+import Foundation
+
+/// Builds the BSON document `MongoBsonAssembly` plans for one JSON text.
+///
+/// Kept apart from `MongoDBConnection` so `scripts/check-mongodb-filter-shapes.sh` compiles this
+/// same code against the libbson the plugin links, rather than a copy of what it is meant to do.
+enum MongoBsonBuilder {
+    /// A new document the caller destroys, or nil after `onParseFailure` has been given libbson's
+    /// reason for the text it refused.
+    static func document(from json: String, onParseFailure: (String) -> Void = { _ in }) -> OpaquePointer? {
+        switch MongoBsonAssembly.plan(json) {
+        case .whole(let text):
+            return parsed(text, onParseFailure: onParseFailure)
+        case .parts(let parts):
+            return assembled(parts, onParseFailure: onParseFailure)
+        }
+    }
+
+    static func errorMessage(_ error: inout bson_error_t) -> String {
+        withUnsafePointer(to: &error.message) { pointer in
+            pointer.withMemoryRebound(to: CChar.self, capacity: 504) { String(cString: $0) }
+        }
+    }
+
+    private static func parsed(_ json: String, onParseFailure: (String) -> Void) -> OpaquePointer? {
+        var error = bson_error_t()
+        let document = json.withCString { bson_new_from_json($0, -1, &error) }
+        if document == nil {
+            onParseFailure(errorMessage(&error))
+        }
+        return document
+    }
+
+    private static func assembled(
+        _ parts: [MongoBsonAssembly.Part],
+        onParseFailure: (String) -> Void
+    ) -> OpaquePointer? {
+        guard let document = bson_new() else { return nil }
+        for part in parts {
+            guard append(part, to: document, onParseFailure: onParseFailure) else {
+                bson_destroy(document)
+                return nil
+            }
+        }
+        return document
+    }
+
+    /// A key is passed with its length, so one holding a NUL is refused rather than cut short.
+    private static func append(
+        _ part: MongoBsonAssembly.Part,
+        to document: OpaquePointer,
+        onParseFailure: (String) -> Void
+    ) -> Bool {
+        switch part {
+        case .member(let json):
+            guard let member = parsed(json, onParseFailure: onParseFailure) else { return false }
+            defer { bson_destroy(member) }
+            return bson_concat(document, member)
+        case .document(let key, let parts):
+            guard let child = assembled(parts, onParseFailure: onParseFailure) else { return false }
+            defer { bson_destroy(child) }
+            return key.withCString { bson_append_document(document, $0, Int32(key.utf8.count), child) }
+        case .array(let key, let parts):
+            guard let child = assembled(parts, onParseFailure: onParseFailure) else { return false }
+            defer { bson_destroy(child) }
+            return key.withCString { bson_append_array(document, $0, Int32(key.utf8.count), child) }
+        }
+    }
+}
+#endif

Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift

@@ -14,9 +14,7 @@ import TableProPluginKit
 #if canImport(CLibMongoc)
 extension MongoDBConnection {
     func bsonErrorMessage(_ error: inout bson_error_t) -> String {
-        withUnsafePointer(to: &error.message) { ptr in
-            ptr.withMemoryRebound(to: CChar.self, capacity: 504) { String(cString: $0) }
-        }
+        MongoBsonBuilder.errorMessage(&error)
     }
 
     func makeError(_ error: bson_error_t) -> MongoDBError {
@@ -519,6 +517,5 @@ extension MongoDBConnection {
         if let cur { mongoc_cursor_destroy(cur) }
         if let col { mongoc_collection_destroy(col) }
     }
-
 }
 #endif

Plugins/MongoDBDriverPlugin/MongoDBConnection.swift

@@ -282,8 +282,8 @@ final class MongoDBConnection: @unchecked Sendable {
             "tls", "tlsAllowInvalidCertificates", "tlsAllowInvalidHostnames",
             "tlsCAFile", "tlsCertificateKeyFile"
         ]
-        if readPreference != nil, !readPreference!.isEmpty { explicitKeys.insert("readPreference") }
-        if writeConcern != nil, !writeConcern!.isEmpty { explicitKeys.insert("w") }
+        if let readPreference, !readPreference.isEmpty { explicitKeys.insert("readPreference") }
+        if let writeConcern, !writeConcern.isEmpty { explicitKeys.insert("w") }
         for (key, value) in extraUriParams where !explicitKeys.contains(key) {
             let encodedValue = value.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? value
             params.append("\(key)=\(encodedValue)")
@@ -925,18 +925,14 @@ final class MongoStreamState: @unchecked Sendable {
 
 extension MongoDBConnection {
     /// Convert a JSON string to a bson_t pointer. Caller must call bson_destroy on the result.
+    ///
+    /// An object opening with `$type`, `$regex` or `$options` becomes the document it is written as,
+    /// the way mongosh sends it, rather than libbson's legacy binary or regular expression value.
     func jsonToBson(_ json: String) -> OpaquePointer? {
         #if canImport(CLibMongoc)
-        var error = bson_error_t()
-
-        // Pass -1 to let bson_new_from_json use strlen on the C string
-        let bson = json.withCString { bson_new_from_json($0, -1, &error) }
-        if bson == nil {
-            var err = error
-            let msg = bsonErrorMessage(&err)
-            logger.debug("Failed to parse JSON to BSON: \(msg)")
+        return MongoBsonBuilder.document(from: json) { message in
+            logger.debug("Failed to parse JSON to BSON: \(message)")
         }
-        return bson
         #else
         return nil
         #endif

Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift

@@ -235,8 +235,9 @@ struct MongoDBQueryBuilder {
             guard ignoresCase else {
                 return MongoDBFilterClause(key: field, body: "{\"$ne\": \(typed(value, kind))}")
             }
-            let body = Self.regexBody(pattern: anchoredPattern(value), ignoresCase: true)
-            return MongoDBFilterClause(key: field, body: "{\"$not\": \(body)}")
+            return MongoDBFilterClause(
+                key: field, body: Self.negatedRegexBody(pattern: anchoredPattern(value), ignoresCase: true)
+            )
         case ">":
             return MongoDBFilterClause(key: field, body: "{\"$gt\": \(typed(value, kind))}")
         case ">=":
@@ -250,8 +251,9 @@ struct MongoDBQueryBuilder {
                 key: field, body: Self.regexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase)
             )
         case "NOT CONTAINS":
-            let body = Self.regexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase)
-            return MongoDBFilterClause(key: field, body: "{\"$not\": \(body)}")
+            return MongoDBFilterClause(
+                key: field, body: Self.negatedRegexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase)
+            )
         case "STARTS WITH":
             let pattern = "^\(escapeRegexChars(value))"
             return MongoDBFilterClause(
@@ -353,6 +355,13 @@ struct MongoDBQueryBuilder {
         return "{\"$regex\": \"\(escapeJsonString(pattern))\", \"$options\": \"i\"}"
     }
 
+    /// `$not` takes a `$regex` operator document only from MongoDB 4.0.7, and a regular expression
+    /// value on every server, so the negated arms send the value.
+    private static func negatedRegexBody(pattern: String, ignoresCase: Bool) -> String {
+        let regex = "{\"pattern\": \"\(escapeJsonString(pattern))\", \"options\": \"\(ignoresCase ? "i" : "")\"}"
+        return "{\"$not\": {\"$regularExpression\": \(regex)}}"
+    }
+
     private func anchoredPattern(_ value: String) -> String {
         "^\(escapeRegexChars(value))$"
     }