diff --git a/.github/workflows/macos-tests.yml b/.github/workflows/macos-tests.yml index 9ddcfac4f..4e19dca44 100644 --- a/.github/workflows/macos-tests.yml +++ b/.github/workflows/macos-tests.yml @@ -183,9 +183,9 @@ jobs: GH_TOKEN: ${{ github.token }} run: scripts/download-libs.sh - # Compiles a C probe against Libs/libbson and parses every filter document the MongoDB - # query builder can emit. It guarded that invariant and ran nowhere; it needs clang and the - # vendored libraries, so this is the first job where it can run at all. + # Compiles the MongoDB driver's own BSON planner and builder against Libs/libbson and builds + # every filter document the query builder can emit, checking the BSON type where it matters. + # It needs swiftc and the vendored libraries, so this is the first job where it can run. - name: Check the MongoDB filter shapes against the query builder run: scripts/check-mongodb-filter-shapes.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index fd43cf0c6..3148cb6c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ⌘F finds and ⇧⌘F toggles filters in data file windows, as in the table grid. - Large data files opened, filtered, sorted and searched in the background, with progress and Cancel. - `.json` and `.ndjson` files opened in the Data Files window rather than as a DuckDB connection. +- `$regex` and `$options` objects in MongoDB scripts and **Raw Filter** sent as operator documents, as in mongosh. ### Removed @@ -529,6 +530,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - MongoDB collections could not be created from **New Table…**. (#3131) - A new or empty MongoDB collection showing only `_id` instead of the fields its validator declares. - MongoDB edits that stored dates and ObjectIds as text, rounded integers past 2^53, or missed a string `_id` that looks numeric. +- MongoDB filters, validators and pipelines with a `$type` or `$regex` object refused as "not a document MongoDB can read". - **New Table…** offered on databases that cannot create a table, such as Redis and Kafka. - Executing indicator and Stop button carried over for a moment onto the query tab switched to. diff --git a/Plugins/MongoDBDriverPlugin/MongoBsonAssembly.swift b/Plugins/MongoDBDriverPlugin/MongoBsonAssembly.swift new file mode 100644 index 000000000..aabdeeb7e --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoBsonAssembly.swift @@ -0,0 +1,146 @@ +import Foundation + +/// How one JSON text becomes one BSON document when libbson's reader would misread part of it. +/// +/// libbson decides what an embedded object is from its first key alone. `$type`, `$regex` and +/// `$options` open its legacy binary and regular expression values, and they are also query +/// operators: `{"sig": {"$type": "binData"}}` fails with `Missing "$binary"`, +/// `{"$regex": "^a", "$exists": true}` fails with `Invalid key "$exists"`, and +/// `{"$regex": "a", "$options": "i"}` becomes a regular expression value. mongosh sends each of them +/// as the document it is written as. So does this: such an object is built member by member, and a +/// member is handed to libbson as a document of its own, `{"key": value}`, where a key is never +/// special. Every value that holds no such object is still read by libbson as written, wrappers +/// such as `{"$oid": …}` included. +enum MongoBsonAssembly: Equatable, Sendable { + /// Text libbson reads as meant, which is nearly all of it. + case whole(String) + /// A document built from these parts, in order. + case parts([Part]) + + enum Part: Equatable, Sendable { + /// A document of one member, which libbson reads as written. + case member(String) + case document(key: String, parts: [Part]) + /// Its parts are keyed `0`, `1` and on, the keys of a BSON array. + case array(key: String, parts: [Part]) + } + + /// libbson's special keys that are also MongoDB operators. + static let operatorKeys: Set = ["$type", "$regex", "$options"] + + /// libbson's other special keys, each opening an Extended JSON value that is never taken apart. + static let valueKeys: Set = [ + "$binary", "$code", "$date", "$dbPointer", "$maxKey", "$minKey", "$numberDecimal", "$numberDouble", + "$numberInt", "$numberLong", "$oid", "$regularExpression", "$scope", "$symbol", "$timestamp", + "$undefined", "$uuid" + ] + + static func plan(_ json: String) -> MongoBsonAssembly { + guard holdsOperatorDocument(json), isWellFormed(json) else { return .whole(json) } + let trimmed = json.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasPrefix("[") { return .parts(elementParts(of: trimmed)) } + return .parts(memberParts(of: trimmed)) + } + + // MARK: - Planning + + private static func memberParts(of objectJson: String) -> [Part] { + MongoScriptJson.members(of: objectJson).map { part(key: $0.key, value: $0.value) } + } + + private static func elementParts(of arrayJson: String) -> [Part] { + MongoScriptJson.topLevelElements(arrayJson).enumerated().map { part(key: String($0.offset), value: $0.element) } + } + + private static func part(key: String, value: String) -> Part { + guard isTakenApart(value) else { return .member("{\(MongoScriptJson.jsonString(key)):\(value)}") } + if value.hasPrefix("[") { return .array(key: key, parts: elementParts(of: value)) } + return .document(key: key, parts: memberParts(of: value)) + } + + /// An operator document is taken apart, and so is anything holding one. A value wrapper is + /// not, whatever it holds: taking a `$code` with a `$scope` apart would store a document where + /// the script wrote code. + private static func isTakenApart(_ valueJson: String) -> Bool { + guard valueJson.hasPrefix("{") || valueJson.hasPrefix("["), + holdsOperatorDocument(valueJson, countingOutermost: true) else { return false } + guard valueJson.hasPrefix("{") else { return true } + let firstKey = MongoScriptJson.members(of: valueJson).first?.key + return firstKey.map { !valueKeys.contains($0) } ?? false + } + + private static func isWellFormed(_ json: String) -> Bool { + (try? JSONSerialization.jsonObject(with: Data(json.utf8), options: .fragmentsAllowed)) != nil + } + + // MARK: - Scanning + + /// Whether an object opens with an operator key once its escapes are decoded, as libbson + /// decodes them. libbson never misreads the outermost object of what it parses, so that one + /// counts only when `countingOutermost` is set. + static func holdsOperatorDocument(_ json: String, countingOutermost: Bool = false) -> Bool { + var text = json + text.makeContiguousUTF8() + let shallowest = countingOutermost ? 1 : 2 + return text.utf8.withContiguousStorageIfAvailable { scan($0, fromDepth: shallowest) } ?? false + } + + private static let quote = UInt8(ascii: "\"") + private static let backslash = UInt8(ascii: "\\") + private static let dollar = UInt8(ascii: "$") + + private static func scan(_ bytes: UnsafeBufferPointer, fromDepth shallowest: Int) -> Bool { + var depth = 0 + var awaitsFirstKey = false + var index = 0 + + while index < bytes.count { + let byte = bytes[index] + switch byte { + case quote: + let end = stringEnd(in: bytes, from: index + 1) + if awaitsFirstKey, depth >= shallowest, isOperatorKey(bytes[(index + 1) ..< end]) { return true } + awaitsFirstKey = false + index = end + case UInt8(ascii: "{"): + depth += 1 + awaitsFirstKey = true + case UInt8(ascii: "["): + depth += 1 + awaitsFirstKey = false + case UInt8(ascii: "}"), UInt8(ascii: "]"): + depth -= 1 + awaitsFirstKey = false + case UInt8(ascii: " "), UInt8(ascii: "\t"), UInt8(ascii: "\n"), UInt8(ascii: "\r"): + break + default: + awaitsFirstKey = false + } + index += 1 + } + return false + } + + /// The index of the quote that closes the string whose contents start at `start`. + private static func stringEnd(in bytes: UnsafeBufferPointer, from start: Int) -> Int { + var index = start + while index < bytes.count { + switch bytes[index] { + case backslash: index += 2 + case quote: return index + default: index += 1 + } + } + return bytes.count + } + + private static let operatorKeyBytes = operatorKeys.map { Array($0.utf8) } + + private static func isOperatorKey(_ raw: Slice>) -> Bool { + guard let first = raw.first, first == dollar || first == backslash else { return false } + guard raw.contains(backslash) else { return operatorKeyBytes.contains { $0.elementsEqual(raw) } } + let quoted = Data([quote] + raw + [quote]) + let decoded = try? JSONSerialization.jsonObject(with: quoted, options: .fragmentsAllowed) as? String + return decoded.map(operatorKeys.contains) ?? false + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoBsonBuilder.swift b/Plugins/MongoDBDriverPlugin/MongoBsonBuilder.swift new file mode 100644 index 000000000..da35c144b --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoBsonBuilder.swift @@ -0,0 +1,77 @@ +// +// MongoBsonBuilder.swift +// MongoDBDriverPlugin +// + +#if canImport(CLibMongoc) +import CLibMongoc +import Foundation + +/// Builds the BSON document `MongoBsonAssembly` plans for one JSON text. +/// +/// Kept apart from `MongoDBConnection` so `scripts/check-mongodb-filter-shapes.sh` compiles this +/// same code against the libbson the plugin links, rather than a copy of what it is meant to do. +enum MongoBsonBuilder { + /// A new document the caller destroys, or nil after `onParseFailure` has been given libbson's + /// reason for the text it refused. + static func document(from json: String, onParseFailure: (String) -> Void = { _ in }) -> OpaquePointer? { + switch MongoBsonAssembly.plan(json) { + case .whole(let text): + return parsed(text, onParseFailure: onParseFailure) + case .parts(let parts): + return assembled(parts, onParseFailure: onParseFailure) + } + } + + static func errorMessage(_ error: inout bson_error_t) -> String { + withUnsafePointer(to: &error.message) { pointer in + pointer.withMemoryRebound(to: CChar.self, capacity: 504) { String(cString: $0) } + } + } + + private static func parsed(_ json: String, onParseFailure: (String) -> Void) -> OpaquePointer? { + var error = bson_error_t() + let document = json.withCString { bson_new_from_json($0, -1, &error) } + if document == nil { + onParseFailure(errorMessage(&error)) + } + return document + } + + private static func assembled( + _ parts: [MongoBsonAssembly.Part], + onParseFailure: (String) -> Void + ) -> OpaquePointer? { + guard let document = bson_new() else { return nil } + for part in parts { + guard append(part, to: document, onParseFailure: onParseFailure) else { + bson_destroy(document) + return nil + } + } + return document + } + + /// A key is passed with its length, so one holding a NUL is refused rather than cut short. + private static func append( + _ part: MongoBsonAssembly.Part, + to document: OpaquePointer, + onParseFailure: (String) -> Void + ) -> Bool { + switch part { + case .member(let json): + guard let member = parsed(json, onParseFailure: onParseFailure) else { return false } + defer { bson_destroy(member) } + return bson_concat(document, member) + case .document(let key, let parts): + guard let child = assembled(parts, onParseFailure: onParseFailure) else { return false } + defer { bson_destroy(child) } + return key.withCString { bson_append_document(document, $0, Int32(key.utf8.count), child) } + case .array(let key, let parts): + guard let child = assembled(parts, onParseFailure: onParseFailure) else { return false } + defer { bson_destroy(child) } + return key.withCString { bson_append_array(document, $0, Int32(key.utf8.count), child) } + } + } +} +#endif diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift index 44282b853..e3715107e 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift @@ -14,9 +14,7 @@ import TableProPluginKit #if canImport(CLibMongoc) extension MongoDBConnection { func bsonErrorMessage(_ error: inout bson_error_t) -> String { - withUnsafePointer(to: &error.message) { ptr in - ptr.withMemoryRebound(to: CChar.self, capacity: 504) { String(cString: $0) } - } + MongoBsonBuilder.errorMessage(&error) } func makeError(_ error: bson_error_t) -> MongoDBError { @@ -519,6 +517,5 @@ extension MongoDBConnection { if let cur { mongoc_cursor_destroy(cur) } if let col { mongoc_collection_destroy(col) } } - } #endif diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift index 310cee6cd..74a428468 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift @@ -282,8 +282,8 @@ final class MongoDBConnection: @unchecked Sendable { "tls", "tlsAllowInvalidCertificates", "tlsAllowInvalidHostnames", "tlsCAFile", "tlsCertificateKeyFile" ] - if readPreference != nil, !readPreference!.isEmpty { explicitKeys.insert("readPreference") } - if writeConcern != nil, !writeConcern!.isEmpty { explicitKeys.insert("w") } + if let readPreference, !readPreference.isEmpty { explicitKeys.insert("readPreference") } + if let writeConcern, !writeConcern.isEmpty { explicitKeys.insert("w") } for (key, value) in extraUriParams where !explicitKeys.contains(key) { let encodedValue = value.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? value params.append("\(key)=\(encodedValue)") @@ -925,18 +925,14 @@ final class MongoStreamState: @unchecked Sendable { extension MongoDBConnection { /// Convert a JSON string to a bson_t pointer. Caller must call bson_destroy on the result. + /// + /// An object opening with `$type`, `$regex` or `$options` becomes the document it is written as, + /// the way mongosh sends it, rather than libbson's legacy binary or regular expression value. func jsonToBson(_ json: String) -> OpaquePointer? { #if canImport(CLibMongoc) - var error = bson_error_t() - - // Pass -1 to let bson_new_from_json use strlen on the C string - let bson = json.withCString { bson_new_from_json($0, -1, &error) } - if bson == nil { - var err = error - let msg = bsonErrorMessage(&err) - logger.debug("Failed to parse JSON to BSON: \(msg)") + return MongoBsonBuilder.document(from: json) { message in + logger.debug("Failed to parse JSON to BSON: \(message)") } - return bson #else return nil #endif diff --git a/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index 0d5b1bad3..d08889a0a 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -235,8 +235,9 @@ struct MongoDBQueryBuilder { guard ignoresCase else { return MongoDBFilterClause(key: field, body: "{\"$ne\": \(typed(value, kind))}") } - let body = Self.regexBody(pattern: anchoredPattern(value), ignoresCase: true) - return MongoDBFilterClause(key: field, body: "{\"$not\": \(body)}") + return MongoDBFilterClause( + key: field, body: Self.negatedRegexBody(pattern: anchoredPattern(value), ignoresCase: true) + ) case ">": return MongoDBFilterClause(key: field, body: "{\"$gt\": \(typed(value, kind))}") case ">=": @@ -250,8 +251,9 @@ struct MongoDBQueryBuilder { key: field, body: Self.regexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase) ) case "NOT CONTAINS": - let body = Self.regexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase) - return MongoDBFilterClause(key: field, body: "{\"$not\": \(body)}") + return MongoDBFilterClause( + key: field, body: Self.negatedRegexBody(pattern: escapeRegexChars(value), ignoresCase: ignoresCase) + ) case "STARTS WITH": let pattern = "^\(escapeRegexChars(value))" return MongoDBFilterClause( @@ -353,6 +355,13 @@ struct MongoDBQueryBuilder { return "{\"$regex\": \"\(escapeJsonString(pattern))\", \"$options\": \"i\"}" } + /// `$not` takes a `$regex` operator document only from MongoDB 4.0.7, and a regular expression + /// value on every server, so the negated arms send the value. + private static func negatedRegexBody(pattern: String, ignoresCase: Bool) -> String { + let regex = "{\"pattern\": \"\(escapeJsonString(pattern))\", \"options\": \"\(ignoresCase ? "i" : "")\"}" + return "{\"$not\": {\"$regularExpression\": \(regex)}}" + } + private func anchoredPattern(_ value: String) -> String { "^\(escapeRegexChars(value))$" } diff --git a/TableProTests/Plugins/MongoBsonAssemblyTests.swift b/TableProTests/Plugins/MongoBsonAssemblyTests.swift new file mode 100644 index 000000000..643a4b6b5 --- /dev/null +++ b/TableProTests/Plugins/MongoBsonAssemblyTests.swift @@ -0,0 +1,194 @@ +// +// MongoBsonAssemblyTests.swift +// TableProTests +// + +import Foundation +import JavaScriptCore +import TableProPluginKit +import Testing + +struct MongoBsonAssemblyTests { + private typealias Part = MongoBsonAssembly.Part + + @Test("Text with no operator document below the root is handed to libbson unchanged") + func textWithoutOperatorDocumentsIsWhole() { + let texts = [ + "{\"status\":\"new\",\"total\":{\"$gt\":{\"$numberInt\":\"5\"}}}", + "[{\"$match\":{\"a\":{\"$numberInt\":\"1\"}}},{\"$limit\":{\"$numberInt\":\"1\"}}]", + "{\"count\":\"t\",\"query\":{\"a\":{\"$exists\":true,\"$type\":\"binData\"}}}", + "{\"$type\":\"binData\"}", + "{\"a\":{\"$binary\":\"AAAA\",\"$type\":\"00\"}}", + "{\"a\":{\"$not\":{\"$regularExpression\":{\"pattern\":\"x\",\"options\":\"i\"}}}}", + "{\"q\":\"{\\\"$type\\\":\\\"binData\\\"}\"}", + "{\"a\":{\"k\":\"$type\"}}", + "{}" + ] + for text in texts { + #expect(MongoBsonAssembly.plan(text) == .whole(text)) + } + } + + @Test("A $type operator becomes a document whose member libbson reads with a literal key") + func typeOperatorBecomesADocument() { + #expect(MongoBsonAssembly.plan("{\"sig\":{\"$type\":\"binData\"}}") == .parts([ + .document(key: "sig", parts: [.member("{\"$type\":\"binData\"}")]) + ])) + #expect(MongoBsonAssembly.plan("{\"tags\":{\"$type\":[\"array\",\"null\"]}}") == .parts([ + .document(key: "tags", parts: [.member("{\"$type\":[\"array\",\"null\"]}")]) + ])) + } + + @Test("A $regex or $options operator becomes a document, with a regular expression value inside kept whole") + func regexOperatorBecomesADocument() { + #expect(MongoBsonAssembly.plan("{\"name\":{\"$regex\":\"^a\",\"$exists\":true}}") == .parts([ + .document(key: "name", parts: [.member("{\"$regex\":\"^a\"}"), .member("{\"$exists\":true}")]) + ])) + #expect(MongoBsonAssembly.plan("{\"name\":{\"$options\":\"i\",\"$regex\":\"^a\"}}") == .parts([ + .document(key: "name", parts: [.member("{\"$options\":\"i\"}"), .member("{\"$regex\":\"^a\"}")]) + ])) + let regexValue = "{\"$regularExpression\":{\"pattern\":\"^a\",\"options\":\"i\"}}" + #expect(MongoBsonAssembly.plan("{\"name\":{\"$regex\":\(regexValue)}}") == .parts([ + .document(key: "name", parts: [.member("{\"$regex\":\(regexValue)}")]) + ])) + } + + @Test("A pipeline is keyed by position, and only the stage holding an operator document is taken apart") + func pipelineElementsAreKeyedByPosition() { + let plan = MongoBsonAssembly.plan( + "[{\"$match\":{\"a\":{\"$type\":\"array\"}}},{\"$limit\":{\"$numberInt\":\"1\"}}]" + ) + #expect(plan == .parts([ + .document(key: "0", parts: [ + .document(key: "$match", parts: [ + .document(key: "a", parts: [.member("{\"$type\":\"array\"}")]) + ]) + ]), + .member("{\"1\":{\"$limit\":{\"$numberInt\":\"1\"}}}") + ])) + } + + @Test("An array holding an operator document becomes a BSON array") + func nestedArrayBecomesAnArray() { + let plan = MongoBsonAssembly.plan("{\"$or\":[{\"a\":{\"$type\":\"string\"}},{\"b\":true}]}") + #expect(plan == .parts([ + .array(key: "$or", parts: [ + .document(key: "0", parts: [.document(key: "a", parts: [.member("{\"$type\":\"string\"}")])]), + .member("{\"1\":{\"b\":true}}") + ]) + ])) + } + + @Test("An escaped operator key is recognised, since libbson decodes a key before reading it") + func escapedOperatorKeyIsRecognised() { + #expect(MongoBsonAssembly.plan("{\"a\":{\"\\u0024type\":\"binData\"}}") == .parts([ + .document(key: "a", parts: [.member("{\"$type\":\"binData\"}")]) + ])) + #expect(MongoBsonAssembly.holdsOperatorDocument("{\"a\":{\"$re\\u0067ex\":\"x\",\"$exists\":true}}")) + } + + @Test("A value wrapper is never taken apart, whatever it holds") + func wrapperIsNeverTakenApart() { + let code = "{\"$code\":\"f\",\"$scope\":{\"a\":{\"$type\":\"binData\"}}}" + #expect(MongoBsonAssembly.plan("{\"x\":\(code)}") == .parts([.member("{\"x\":\(code)}")])) + } + + @Test("Malformed text is left to libbson rather than cut short") + func malformedTextIsWhole() { + let unclosed = "{\"a\":{\"$type\":\"binData\"}" + #expect(MongoBsonAssembly.plan(unclosed) == .whole(unclosed)) + } + + @Test("Every other member keeps its exact text, duplicates included") + func membersKeepTheirExactText() { + let plan = MongoBsonAssembly.plan( + "{\"a\":{\"$type\":\"double\"},\"n\":{\"$numberLong\":\"9007199254740993\"},\"f\":1.0,\"a\":2}" + ) + #expect(plan == .parts([ + .document(key: "a", parts: [.member("{\"$type\":\"double\"}")]), + .member("{\"n\":{\"$numberLong\":\"9007199254740993\"}}"), + .member("{\"f\":1.0}"), + .member("{\"a\":2}") + ])) + } + + @Test("Joining every part back gives the text that was planned, in order") + func partsJoinBackToTheInput() throws { + let texts = [ + "{\"name\":{\"$regex\":{\"$regularExpression\":{\"pattern\":\"^a\",\"options\":\"i\"}}},\"n\":1}", + "[{\"$match\":{\"sig\":{\"$type\":\"binData\"},\"x\":[1,{\"y\":{\"$options\":\"i\",\"$regex\":\"z\"}}]}}]", + "{\"create\":\"v\",\"validator\":{\"$or\":[{\"a\":{\"$type\":\"double\"}},{\"a\":null}]},\"a\":{\"$b\":1}}", + "{\"_id\":{\"$oid\":\"507f1f77bcf86cd799439011\"},\"m\":{\"$type\":\"x\",\"k\":[]},\"m\":\"dup\"}" + ] + for text in texts { + guard case .parts(let parts) = MongoBsonAssembly.plan(text) else { + Issue.record("\(text) was not taken apart") + continue + } + let isArray = text.hasPrefix("[") + let body = try joined(parts, isArray: isArray) + #expect((isArray ? "[\(body)]" : "{\(body)}") == text) + } + } + + @Test("The shell's own serialization of an operator document reaches the planner as a document") + func shellOutputIsTakenApart() throws { + let host = MongoScriptPreludeTests.RecordingHost() + host.replies = ["1", "2", "{\"insertedIds\": [{\"$oid\": \"507f1f77bcf86cd799439011\"}], \"insertedCount\": 1}"] + let context = try MongoScriptContext.make( + execute: { host.handle($0) }, + emit: { host.record(printed: $0) } + ) + + context.evaluateScript("db.t.find({sig: {$type: \"binData\"}}); db.t.find({name: {$regex: /^a/i}})") + context.evaluateScript("db.t.insertOne({m: {$regex: \"a\", $options: \"i\"}})") + #expect(context.exception == nil) + + let filters = host.requests(op: "openCursor").compactMap { $0["filter"] as? String } + let documents = host.requests(op: "insertOne").compactMap { $0["document"] as? String } + #expect(filters.count == 2) + #expect(documents.count == 1) + for text in filters + documents { + #expect(MongoBsonAssembly.plan(text) != .whole(text)) + } + } + + @Test("A raw filter row and a grid regex reach the count as the same document the rows query sends") + func filterBarOutputIsTakenApart() throws { + let raw = try #require(MongoDBRawFilterNormalizer().normalize("{sig: {$type: \"binData\"}}")) + #expect(MongoBsonAssembly.plan(raw) != .whole(raw)) + + let builder = MongoDBQueryBuilder() + let contains = builder.buildFilterDocument(from: [ + PluginQueryFilter(column: "name", op: "CONTAINS", value: "a", isCaseSensitive: false) + ]) + let notContains = builder.buildFilterDocument(from: [ + PluginQueryFilter(column: "name", op: "NOT CONTAINS", value: "a", isCaseSensitive: false) + ]) + #expect(MongoBsonAssembly.plan(contains) != .whole(contains)) + #expect(MongoBsonAssembly.plan(notContains) == .whole(notContains)) + } + + private func joined(_ parts: [Part], isArray: Bool) throws -> String { + try parts.enumerated().map { index, part in + let (key, value) = try keyAndValue(of: part) + guard isArray else { return "\(MongoScriptJson.jsonString(key)):\(value)" } + #expect(key == String(index)) + return value + }.joined(separator: ",") + } + + private func keyAndValue(of part: Part) throws -> (key: String, value: String) { + switch part { + case .member(let json): + let members = MongoScriptJson.members(of: json) + #expect(members.count == 1) + let only = try #require(members.first) + return (only.key, only.value) + case .document(let key, let parts): + return (key, "{\(try joined(parts, isArray: false))}") + case .array(let key, let parts): + return (key, "[\(try joined(parts, isArray: true))]") + } + } +} diff --git a/TableProTests/Plugins/MongoDBQueryBuilderTests.swift b/TableProTests/Plugins/MongoDBQueryBuilderTests.swift index 1f026bbf0..68e750000 100644 --- a/TableProTests/Plugins/MongoDBQueryBuilderTests.swift +++ b/TableProTests/Plugins/MongoDBQueryBuilderTests.swift @@ -225,8 +225,40 @@ struct MongoDBQueryBuilderTests { collection: "users", queryFilters: [PluginQueryFilter(column: "name", op: "NOT CONTAINS", value: "test")] ) - #expect(query.contains("\"$not\"")) - #expect(query.contains("\"$regex\": \"test\"")) + #expect(query.contains( + "{\"name\": {\"$not\": {\"$regularExpression\": {\"pattern\": \"test\", \"options\": \"\"}}}}" + )) + } + + @Test("A negated match sends a regular expression value, which $not takes on every server") + func negatedMatchesSendARegularExpressionValue() { + let notContains = parseFilter(builder.buildFilterDocument(from: [ + PluginQueryFilter(column: "name", op: "NOT CONTAINS", value: "a.b", isCaseSensitive: false) + ])) + let notEqual = parseFilter(builder.buildFilterDocument(from: [ + PluginQueryFilter(column: "name", op: "!=", value: "Alice", isCaseSensitive: false) + ])) + + let containsRegex = negatedRegex(in: notContains, field: "name") + #expect(containsRegex?["pattern"] as? String == "a\\.b") + #expect(containsRegex?["options"] as? String == "i") + let equalRegex = negatedRegex(in: notEqual, field: "name") + #expect(equalRegex?["pattern"] as? String == "^Alice$") + #expect(equalRegex?["options"] as? String == "i") + } + + @Test("A positive match keeps the operator form mongosh reads") + func positiveMatchesKeepTheOperatorForm() { + let doc = builder.buildFilterDocument(from: [ + PluginQueryFilter(column: "name", op: "CONTAINS", value: "ali", isCaseSensitive: false) + ]) + #expect(doc == "{\"name\": {\"$regex\": \"ali\", \"$options\": \"i\"}}") + } + + private func negatedRegex(in filter: [String: Any]?, field: String) -> [String: Any]? { + let condition = filter?[field] as? [String: Any] + let negated = condition?["$not"] as? [String: Any] + return negated?["$regularExpression"] as? [String: Any] } @Test("Filtered query with STARTS WITH operator") @@ -694,8 +726,7 @@ struct MongoDBQueryBuilderTests { ) #expect(doc != nil) #expect(doc.map { Array($0.keys) } == ["name"]) - let not = (doc?["name"] as? [String: Any])?["$not"] as? [String: Any] - #expect((not?["$regex"] as? String)?.contains("$where") == true) + #expect((negatedRegex(in: doc, field: "name")?["pattern"] as? String)?.contains("$where") == true) } @Test("STARTS WITH escapes embedded double quotes as data") diff --git a/docs/databases/mongodb.mdx b/docs/databases/mongodb.mdx index 8145037bd..f4d63c6ec 100644 --- a/docs/databases/mongodb.mdx +++ b/docs/databases/mongodb.mdx @@ -128,6 +128,13 @@ available: object literals with unquoted keys, single-quoted strings, regex lite return, so a filter written that way keeps matching. `new Date` and `instanceof Date` are the native ones. +`$type`, `$regex` and `$options` objects reach the server as the operator documents they are +written as, the same as in mongosh: `{tags: {$type: ["array", "null"]}}` and +`{name: {$regex: "^a", $options: "i"}}` filter in a query tab and in the filter bar's +**Raw Filter** row. Such an object is a document everywhere, so `insertOne` stores +`{$regex: "a", $options: "i"}` as an embedded document and MongoDB refuses it inside `$in`. Write +a regular expression value as a literal, `/^a/i`. + ```javascript db.orders.find({status: "completed"}, {customerId: 1, total: 1}) .sort({date: -1}) @@ -217,6 +224,7 @@ New connections default to **Disabled**, and the driver has no TLS fallback: **P - Nested paths filter but do not sort. Sorting works on the grid's own columns. - **same element** covers a field one array deep. A path through an array inside another array needs nested `$elemMatch`, so those filter with dot notation only. - GridFS buckets are not browsable, and change streams are unsupported. +- A `Code` scope holding an object that opens with `$type`, `$regex` or `$options`, such as `Code("f", {a: {$type: "binData"}})`, fails with `This is not a document MongoDB can read`. List another key of that object first. - A script that loops without touching the database cannot be stopped: JavaScriptCore has no public way to interrupt one. `Cmd+.` stops anything that reads, writes or prints, which covers every query. A script silent for 120 seconds is abandoned and the shell restarts. - Field names that look like integers (`"0"`, `"12"`) sort ahead of the rest in a document literal, which is what JavaScript does with them. diff --git a/project.yml b/project.yml index ecd82d75d..0db783c05 100644 --- a/project.yml +++ b/project.yml @@ -505,6 +505,7 @@ targets: - Plugins/MSSQLDriverPlugin/MSSQLSessionTransaction.swift - Plugins/MQLExportPlugin/MQLExportHelpers.swift - Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift + - Plugins/MongoDBDriverPlugin/MongoBsonAssembly.swift - Plugins/MongoDBDriverPlugin/MongoDBDecimal128.swift - Plugins/MongoDBDriverPlugin/MongoDBAuthSourceResolver.swift - Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift diff --git a/scripts/check-mongodb-filter-shapes.sh b/scripts/check-mongodb-filter-shapes.sh index 71041d54e..23cc301d4 100755 --- a/scripts/check-mongodb-filter-shapes.sh +++ b/scripts/check-mongodb-filter-shapes.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash # -# Every filter document MongoDBQueryBuilder can emit, parsed by the libbson we actually link. +# Every filter document MongoDBQueryBuilder can emit, built into BSON by the MongoDB driver's own +# MongoBsonBuilder against the libbson we actually link. # -# The builder assembles Extended JSON by hand, and MongoDBConnection hands the result to -# bson_new_from_json. A shape that stops parsing there fails at run time with an unhelpful -# error and no test catches it, because the Swift tests only compare strings. Run this after -# bumping libbson, or after adding an operator arm that emits a new shape. +# The builder assembles Extended JSON by hand. MongoBsonBuilder hands the text to +# bson_new_from_json whole, or builds it member by member when it holds a $type, $regex or +# $options operator document, which libbson would otherwise read as a binary or regular +# expression value. A shape that stops building fails at run time with an unhelpful error, and one +# built as the wrong BSON type matches something else without failing, while the Swift tests only +# compare strings. So this compiles the plugin's planner and builder with swiftc, builds every +# shape, and checks the BSON type wherever libbson's own reading and the query's meaning part. +# Run it after bumping libbson, or after adding an operator arm that emits a new shape. # # Usage: scripts/check-mongodb-filter-shapes.sh [arm64|x86_64] @@ -13,8 +18,9 @@ set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" ARCH="${1:-$(uname -m)}" +PLUGIN="$ROOT/Plugins/MongoDBDriverPlugin" LIB="$ROOT/Libs/libbson_${ARCH}.a" -INCLUDE="$ROOT/Plugins/MongoDBDriverPlugin/CLibMongoc/include" +INCLUDE="$PLUGIN/CLibMongoc/include" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT @@ -23,66 +29,183 @@ if [ ! -f "$LIB" ]; then exit 2 fi -cat > "$WORK/probe.c" <<'PROBE' -#include -#include +cat > "$WORK/main.swift" <<'PROBE' +import CLibMongoc +import Foundation -static int failures = 0; +/// What libbson stored a value as, read back from its canonical Extended JSON, which spells every +/// type but a document, an array, a string, a boolean and null as a one-key wrapper. +enum Kind: String { + case document, array, string, bool, null, regex, binary, date, objectId, decimal, other -static void check(const char *label, const char *json) { - bson_error_t error; - bson_t *doc = bson_new_from_json((const uint8_t *)json, -1, &error); - if (!doc) { - printf("FAIL %-34s %s\n %s\n", label, error.message, json); - failures++; - return; + init(_ value: Any) { + switch value { + case is String: self = .string + case is NSNull: self = .null + case is [Any]: self = .array + case let number as NSNumber where CFGetTypeID(number) == CFBooleanGetTypeID(): self = .bool + case let object as [String: Any]: + let wrappers: [String: Kind] = [ + "$regularExpression": .regex, "$binary": .binary, "$date": .date, "$oid": .objectId, + "$numberDecimal": .decimal, "$numberInt": .other, "$numberLong": .other, "$numberDouble": .other, + "$timestamp": .other, "$code": .other, "$symbol": .other, "$minKey": .other, "$maxKey": .other, + "$undefined": .other, "$dbPointer": .other + ] + guard object.count == 1, let key = object.keys.first, let kind = wrappers[key] else { + self = .document + return + } + self = kind + default: self = .other + } } - printf("ok %-34s\n", label); - bson_destroy(doc); } -int main(void) { - printf("libbson %s\n", bson_get_version()); +struct ShapeCheck { + private(set) var failures = 0 - check("dotted equality", "{\"customer.country\": \"US\"}"); - check("dotted comparison", "{\"customer.age\": {\"$gte\": 18}}"); - check("array dot notation", "{\"items.sku\": \"A100\"}"); - check("elemMatch multi", "{\"items\": {\"$elemMatch\": {\"price\": {\"$gt\": 500}, \"name\": \"Laptop\"}}}"); - check("elemMatch single", "{\"items\": {\"$elemMatch\": {\"sku\": \"A100\"}}}"); - check("elemMatch dotted inner", "{\"orders\": {\"$elemMatch\": {\"customer.country\": \"US\"}}}"); - check("elemMatch colliding key","{\"items\": {\"$elemMatch\": {\"$and\": [{\"price\": {\"$gt\": 10}}, {\"price\": {\"$lt\": 90}}]}}}"); - check("elemMatch match-any", "{\"items\": {\"$elemMatch\": {\"$or\": [{\"price\": {\"$gt\": 500}}, {\"name\": \"Laptop\"}]}}}"); - check("elemMatch non-BMP key", "{\"🎁items\": {\"$elemMatch\": {\"sku\": \"A100\"}}}"); - check("string field quoted", "{\"customer.zip\": \"12345\"}"); - check("elemMatch regex", "{\"items\": {\"$elemMatch\": {\"name\": {\"$regex\": \"^Lap\", \"$options\": \"i\"}}}}"); - check("elemMatch not regex", "{\"items\": {\"$elemMatch\": {\"name\": {\"$not\": {\"$regex\": \"^Lap\"}}}}}"); - check("and of clauses", "{\"$and\": [{\"items.sku\": \"A100\"}, {\"customer.country\": \"US\"}]}"); - check("or of clauses", "{\"$or\": [{\"items\": {\"$elemMatch\": {\"price\": {\"$gt\": 500}}}}, {\"a\": 1}]}"); - check("raw wrapped", "{\"$and\": [{\"customer.country\": \"US\"}]}"); - check("raw match all", "{\"$and\": [{}]}"); - check("impossible filter", "{\"_id\": {\"$in\": []}}"); - check("date coercion", "{\"createdAt\": {\"$gte\": {\"$date\": {\"$numberLong\": \"1704067200000\"}}}}"); - check("objectId coercion", "{\"_id\": {\"$gt\": {\"$oid\": \"507f1f77bcf86cd799439011\"}}}"); - check("decimal coercion", "{\"price\": {\"$gte\": {\"$numberDecimal\": \"19.99\"}}}"); - check("objectId both ways", "{\"$or\": [{\"ref\": {\"$oid\": \"507f1f77bcf86cd799439011\"}}, {\"ref\": \"507f1f77bcf86cd799439011\"}]}"); - check("binary wrapper", "{\"id\": {\"$binary\": {\"base64\": \"TGVnYWN5AAAAAAAAAAAAAA==\", \"subType\": \"03\"}}}"); - check("regex no options", "{\"name\": {\"$regex\": \"^A\"}}"); - check("ne null", "{\"name\": {\"$ne\": null}}"); - check("nor list", "{\"$nor\": [{\"a\": {\"$regex\": \"^x$\", \"$options\": \"i\"}}]}"); - check("between", "{\"name\": {\"$gte\": \"Smith, John\", \"$lte\": \"Zed\"}}"); - check("getField escape hatch", "{\"$expr\": {\"$gt\": [{\"$getField\": \"price.usd\"}, 40]}}"); + mutating func check(_ label: String, _ json: String, kinds: KeyValuePairs = [:]) { + var reason = "libbson refused it" + guard let document = MongoBsonBuilder.document(from: json, onParseFailure: { reason = $0 }) else { + report(label, "does not build: \(reason)", json) + return + } + defer { bson_destroy(document) } + guard let text = bson_as_canonical_extended_json(document, nil) else { + report(label, "has no canonical Extended JSON", json) + return + } + defer { bson_free(text) } + let built = try? JSONSerialization.jsonObject(with: Data(String(cString: text).utf8)) + for (path, expected) in kinds { + guard let value = Self.value(at: path, in: built) else { + report(label, "has no \(path)", json) + return + } + let actual = Kind(value) + guard actual == expected else { + report(label, "holds \(path) as \(actual.rawValue), not \(expected.rawValue)", json) + return + } + } + print("ok \(label)") + } + + private static func value(at path: String, in root: Any?) -> Any? { + path.split(separator: ".").reduce(root) { node, key in + if let object = node as? [String: Any] { return object[String(key)] } + if let array = node as? [Any], let index = Int(key), array.indices.contains(index) { return array[index] } + return nil + } + } - if (failures) { - printf("\n%d shape(s) failed to parse\n", failures); - return 1; + private mutating func report(_ label: String, _ problem: String, _ json: String) { + print("FAIL \(label): \(problem)\n \(json)") + failures += 1 } - printf("\nall shapes parse\n"); - return 0; } + +print("libbson \(String(cString: bson_get_version()))") +var shapes = ShapeCheck() + +shapes.check("dotted equality", #"{"customer.country": "US"}"#) +shapes.check("dotted comparison", #"{"customer.age": {"$gte": 18}}"#) +shapes.check("array dot notation", #"{"items.sku": "A100"}"#) +shapes.check("elemMatch multi", #"{"items": {"$elemMatch": {"price": {"$gt": 500}, "name": "Laptop"}}}"#) +shapes.check("elemMatch single", #"{"items": {"$elemMatch": {"sku": "A100"}}}"#) +shapes.check("elemMatch dotted inner", #"{"orders": {"$elemMatch": {"customer.country": "US"}}}"#) +shapes.check( + "elemMatch colliding key", + #"{"items": {"$elemMatch": {"$and": [{"price": {"$gt": 10}}, {"price": {"$lt": 90}}]}}}"# +) +shapes.check( + "elemMatch match-any", + #"{"items": {"$elemMatch": {"$or": [{"price": {"$gt": 500}}, {"name": "Laptop"}]}}}"# +) +shapes.check("elemMatch non-BMP key", #"{"🎁items": {"$elemMatch": {"sku": "A100"}}}"#) +shapes.check("string field quoted", #"{"customer.zip": "12345"}"#) +shapes.check( + "elemMatch regex", + #"{"items": {"$elemMatch": {"name": {"$regex": "^Lap", "$options": "i"}}}}"#, + kinds: ["items.$elemMatch.name": .document, "items.$elemMatch.name.$regex": .string] +) +shapes.check( + "elemMatch not regex", + #"{"items": {"$elemMatch": {"name": {"$not": {"$regularExpression": {"pattern": "^Lap", "options": ""}}}}}}"#, + kinds: ["items.$elemMatch.name.$not": .regex] +) +shapes.check( + "not regex ignoring case", + #"{"name": {"$not": {"$regularExpression": {"pattern": "^a$", "options": "i"}}}}"#, + kinds: ["name.$not": .regex] +) +shapes.check("and of clauses", #"{"$and": [{"items.sku": "A100"}, {"customer.country": "US"}]}"#) +shapes.check("or of clauses", #"{"$or": [{"items": {"$elemMatch": {"price": {"$gt": 500}}}}, {"a": 1}]}"#) +shapes.check("raw wrapped", #"{"$and": [{"customer.country": "US"}]}"#) +shapes.check("raw match all", #"{"$and": [{}]}"#) +shapes.check("impossible filter", #"{"_id": {"$in": []}}"#) +shapes.check( + "date coercion", + #"{"createdAt": {"$gte": {"$date": {"$numberLong": "1704067200000"}}}}"#, + kinds: ["createdAt.$gte": .date] +) +shapes.check( + "objectId coercion", + #"{"_id": {"$gt": {"$oid": "507f1f77bcf86cd799439011"}}}"#, + kinds: ["_id.$gt": .objectId] +) +shapes.check( + "decimal coercion", + #"{"price": {"$gte": {"$numberDecimal": "19.99"}}}"#, + kinds: ["price.$gte": .decimal] +) +shapes.check( + "objectId both ways", + #"{"$or": [{"ref": {"$oid": "507f1f77bcf86cd799439011"}}, {"ref": "507f1f77bcf86cd799439011"}]}"#, + kinds: ["$or.0.ref": .objectId, "$or.1.ref": .string] +) +shapes.check( + "binary wrapper", + #"{"id": {"$binary": {"base64": "TGVnYWN5AAAAAAAAAAAAAA==", "subType": "03"}}}"#, + kinds: ["id": .binary] +) +shapes.check( + "regex no options", + #"{"name": {"$regex": "^A"}}"#, + kinds: ["name": .document, "name.$regex": .string] +) +shapes.check("ne null", #"{"name": {"$ne": null}}"#, kinds: ["name.$ne": .null]) +shapes.check( + "nor list", + #"{"$nor": [{"a": {"$regex": "^x$", "$options": "i"}}]}"#, + kinds: ["$nor": .array, "$nor.0.a": .document, "$nor.0.a.$options": .string] +) +shapes.check("between", #"{"name": {"$gte": "Smith, John", "$lte": "Zed"}}"#) +shapes.check("getField escape hatch", #"{"$expr": {"$gt": [{"$getField": "price.usd"}, 40]}}"#) +shapes.check( + "raw type operator", + #"{"sig": {"$type": "binData"}}"#, + kinds: ["sig": .document, "sig.$type": .string] +) +shapes.check( + "raw regex beside another operator", + #"{"f": {"$regex": "^a", "$exists": true}}"#, + kinds: ["f": .document, "f.$exists": .bool] +) + +guard shapes.failures == 0 else { + print("\n\(shapes.failures) shape(s) failed") + exit(1) +} +print("\nevery shape builds as the query means it") PROBE -clang -arch "$ARCH" -I "$INCLUDE" "$WORK/probe.c" "$LIB" \ - -lresolv -framework CoreFoundation -framework Security \ +xcrun --sdk macosx swiftc -swift-version 6 -Onone -module-name FilterShapes \ + -target "${ARCH}-apple-macos14.0" \ + -I "$PLUGIN/CLibMongoc" \ + -Xcc -I"$INCLUDE" -Xcc -I"$INCLUDE/libbson-1.0" -Xcc -I"$INCLUDE/libmongoc-1.0" \ + "$PLUGIN/MongoScriptJson.swift" "$PLUGIN/MongoBsonAssembly.swift" "$PLUGIN/MongoBsonBuilder.swift" \ + "$WORK/main.swift" "$LIB" -lresolv -framework CoreFoundation -framework Security \ -o "$WORK/probe" || exit 2 "$WORK/probe"