From 06717cf9021918c7e3af8c437444c8554f36de4c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 26 Sep 2026 22:03:37 +0700 Subject: [PATCH] fix(plugins): keep server-provided names inside comments and string literals in MQL exports --- CHANGELOG.md | 1 + Packages/TableProCore/Package.swift | 11 + .../JavaScriptText.swift | 118 ++++++++ .../JavaScriptTextTests.swift | 110 +++++++ .../PluginExportUtilitiesTests.swift | 56 ++++ .../MQLCollectionDefinition.swift | 141 +++++++++ .../MQLExportPlugin/MQLExportHelpers.swift | 46 ++- Plugins/MQLExportPlugin/MQLExportPlugin.swift | 42 +-- Plugins/MQLExportPlugin/MQLScriptLexer.swift | 226 ++++++++++++++ Plugins/MQLExportPlugin/MQLScriptValue.swift | 177 +++++++++++ .../PluginExportUtilities.swift | 23 +- TablePro/Models/Query/QueryTab.swift | 3 +- .../Models/Query/QueryTabBaseQueryTests.swift | 11 + .../MQLCollectionDefinitionTests.swift | 284 ++++++++++++++++++ .../Plugins/MQLExportHelpersTests.swift | 54 ++++ docs/features/import-export.mdx | 2 + project.yml | 11 +- 17 files changed, 1271 insertions(+), 45 deletions(-) create mode 100644 Packages/TableProCore/Sources/TableProJavaScriptText/JavaScriptText.swift create mode 100644 Packages/TableProCore/Tests/TableProJavaScriptTextTests/JavaScriptTextTests.swift create mode 100644 Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportUtilitiesTests.swift create mode 100644 Plugins/MQLExportPlugin/MQLCollectionDefinition.swift create mode 100644 Plugins/MQLExportPlugin/MQLScriptLexer.swift create mode 100644 Plugins/MQLExportPlugin/MQLScriptValue.swift create mode 100644 TableProTests/Plugins/MQLCollectionDefinitionTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index fd43cf0c69..b25b8a8013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -559,6 +559,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - SQL Server connections set to Required (skip verify) not encrypted past the login. - Inline suggestions sending the query and table columns to the AI provider on Ask Each Time and Never connections. - Stored MongoDB values and collection names that ran as shell code when a row was edited, duplicated or restored. +- Server-provided collection, database and index names that could break out of comments and strings in an MQL export. (#3132) ## [0.75.0] - 2026-09-18 diff --git a/Packages/TableProCore/Package.swift b/Packages/TableProCore/Package.swift index e68a35b818..e944b5a19e 100644 --- a/Packages/TableProCore/Package.swift +++ b/Packages/TableProCore/Package.swift @@ -27,6 +27,7 @@ let package = Package( .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), .library(name: "TableProDocumentPath", targets: ["TableProDocumentPath"]), + .library(name: "TableProJavaScriptText", targets: ["TableProJavaScriptText"]), .library(name: "TableProLogRedaction", targets: ["TableProLogRedaction"]), .library(name: "TableProR2SQLCore", targets: ["TableProR2SQLCore"]), .library(name: "TableProConnectionLibrary", targets: ["TableProConnectionLibrary"]), @@ -48,6 +49,11 @@ let package = Package( dependencies: [], path: "Sources/TableProDocumentPath" ), + .target( + name: "TableProJavaScriptText", + dependencies: [], + path: "Sources/TableProJavaScriptText" + ), .target( name: "TableProLogRedaction", dependencies: [], @@ -213,6 +219,11 @@ let package = Package( dependencies: ["TableProDocumentPath"], path: "Tests/TableProDocumentPathTests" ), + .testTarget( + name: "TableProJavaScriptTextTests", + dependencies: ["TableProJavaScriptText"], + path: "Tests/TableProJavaScriptTextTests" + ), .testTarget( name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"], diff --git a/Packages/TableProCore/Sources/TableProJavaScriptText/JavaScriptText.swift b/Packages/TableProCore/Sources/TableProJavaScriptText/JavaScriptText.swift new file mode 100644 index 0000000000..008305eafc --- /dev/null +++ b/Packages/TableProCore/Sources/TableProJavaScriptText/JavaScriptText.swift @@ -0,0 +1,118 @@ +// +// JavaScriptText.swift +// TableProJavaScriptText +// + +import Foundation + +/// Shell text written around a name the server chose, each piece escaped for where it stands. +/// +/// JSON only requires the C0 controls to be escaped inside a string. JavaScript also ends a line +/// at U+2028 and U+2029, and the editor's statement scanner ends one wherever +/// `Character.isNewline` does, which adds U+0085, VT and FF. Written raw, any of them ended the +/// comment or the string around a name early, and the rest of the name was read as script. +public enum JavaScriptText { + private static let quote = UInt8(ascii: "\"") + private static let backslash = UInt8(ascii: "\\") + + /// The escape for a character that ends a line or cannot be seen, or nil for any other. + public static func lineBreakingEscape(_ scalar: Unicode.Scalar) -> String? { + switch scalar { + case "\n": return "\\n" + case "\r": return "\\r" + case "\t": return "\\t" + default: + let value = scalar.value + guard value < 0x20 || (0x7F ... 0x9F).contains(value) || value == 0x2028 || value == 0x2029 else { + return nil + } + return String(format: "\\u%04x", value) + } + } + + /// A double-quoted string literal on one line, which JSON and JavaScript both read back as `value`. + public static func stringLiteral(_ value: String) -> String { + var output: [UInt8] = [quote] + output.reserveCapacity(value.utf8.count + 2) + let wasContiguous = value.utf8.withContiguousStorageIfAvailable { appendEscaped($0, to: &output) } != nil + if !wasContiguous { + Array(value.utf8).withUnsafeBufferPointer { appendEscaped($0, to: &output) } + } + output.append(quote) + return String(decoding: output, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + } + + /// A `//` comment that ends where its own line does. + public static func lineComment(_ text: String) -> String { + var line = "//" + guard !text.isEmpty else { return line } + line.append(" ") + for scalar in text.unicodeScalars { + if let escape = lineBreakingEscape(scalar) { + line.append(escape) + } else { + line.unicodeScalars.append(scalar) + } + } + return line + } + + /// Whether `name` can follow a `.` as it is: ASCII letters, digits and `_`, not led by a digit. + /// + /// Checked byte by byte, because a `Character` is a whole grapheme cluster, and a Unicode + /// Prepend letter joined to a `(` or `;` answers `isLetter` for the pair. + public static func isPlainIdentifier(_ name: String) -> Bool { + guard let first = name.utf8.first, !isASCIIDigit(first) else { return false } + return name.utf8.allSatisfy { isASCIILetter($0) || isASCIIDigit($0) || $0 == UInt8(ascii: "_") } + } + + private static func isASCIILetter(_ byte: UInt8) -> Bool { + (UInt8(ascii: "a") ... UInt8(ascii: "z")).contains(byte) || (UInt8(ascii: "A") ... UInt8(ascii: "Z")).contains(byte) + } + + private static func isASCIIDigit(_ byte: UInt8) -> Bool { + (UInt8(ascii: "0") ... UInt8(ascii: "9")).contains(byte) + } + + /// Runs on the UTF-8 bytes, because an export calls it once per value and a loop over + /// `unicodeScalars` measured twice as slow. Every other byte is copied as it is. + private static func appendEscaped(_ bytes: UnsafeBufferPointer, to output: inout [UInt8]) { + var index = 0 + while index < bytes.count { + let byte = bytes[index] + if byte == quote || byte == backslash { + output.append(backslash) + output.append(byte) + index += 1 + } else if let sequence = escapableSequence(in: bytes, at: index), + let escape = lineBreakingEscape(sequence.scalar) { + output.append(contentsOf: escape.utf8) + index += sequence.width + } else { + output.append(byte) + index += 1 + } + } + } + + /// The character at `index` when its lead byte is one that can spell a character + /// `lineBreakingEscape` answers for: a C0 control or DEL, 0xC2 for U+0080 to U+00BF, or 0xE2 + /// for U+2000 to U+2FFF. A Swift string is valid UTF-8, so the continuation bytes are there. + private static func escapableSequence( + in bytes: UnsafeBufferPointer, + at index: Int + ) -> (scalar: Unicode.Scalar, width: Int)? { + let lead = bytes[index] + switch lead { + case 0x00 ..< 0x20, 0x7F: + return (Unicode.Scalar(lead), 1) + case 0xC2 where index + 1 < bytes.count: + return (Unicode.Scalar(bytes[index + 1]), 2) + case 0xE2 where index + 2 < bytes.count: + let value = 0x2000 | UInt32(bytes[index + 1] & 0x3F) << 6 | UInt32(bytes[index + 2] & 0x3F) + return Unicode.Scalar(value).map { (scalar: $0, width: 3) } + default: + return nil + } + } +} diff --git a/Packages/TableProCore/Tests/TableProJavaScriptTextTests/JavaScriptTextTests.swift b/Packages/TableProCore/Tests/TableProJavaScriptTextTests/JavaScriptTextTests.swift new file mode 100644 index 0000000000..8c37211b69 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProJavaScriptTextTests/JavaScriptTextTests.swift @@ -0,0 +1,110 @@ +// +// JavaScriptTextTests.swift +// TableProJavaScriptTextTests +// + +import Foundation +import JavaScriptCore +import Testing + +@testable import TableProJavaScriptText + +struct JavaScriptTextTests { + private static let lineTerminators: [Unicode.Scalar] = [ + "\n", "\r", "\u{0B}", "\u{0C}", "\u{85}", "\u{2028}", "\u{2029}" + ] + + private static let mixedName = "a\u{2028}b\u{2029}c\u{85}d\u{0B}e\u{0C}f\"g\\h\ni\rj\tk\u{7F}l\u{9F}m\u{00}n😀o" + + private static func containsRawLineTerminator(_ text: String) -> Bool { + text.unicodeScalars.contains { lineTerminators.contains($0) } + } + + @Test("Each line-breaking character has the escape the MongoDB driver's statements use") + func escapeSpelling() { + let expected: [(Unicode.Scalar, String)] = [ + ("\n", "\\n"), ("\r", "\\r"), ("\t", "\\t"), ("\u{00}", "\\u0000"), ("\u{0B}", "\\u000b"), + ("\u{0C}", "\\u000c"), ("\u{1F}", "\\u001f"), ("\u{7F}", "\\u007f"), ("\u{85}", "\\u0085"), + ("\u{9F}", "\\u009f"), ("\u{2028}", "\\u2028"), ("\u{2029}", "\\u2029") + ] + for (scalar, escape) in expected { + #expect(JavaScriptText.lineBreakingEscape(scalar) == escape, "U+\(String(scalar.value, radix: 16))") + } + for scalar: Unicode.Scalar in ["a", " ", "\"", "\\", "\u{A0}", "\u{E9}", "\u{2027}", "\u{202A}", "\u{1F600}"] { + #expect(JavaScriptText.lineBreakingEscape(scalar) == nil, "U+\(String(scalar.value, radix: 16))") + } + } + + /// The literal is built from UTF-8 bytes for speed, so it has to agree with the scalar rule + /// for every character, not only the ones picked for a test. + @Test("A string literal escapes exactly what the scalar rule escapes, for every BMP character") + func literalAgreesWithScalarRuleEverywhere() { + var mismatches: [UInt32] = [] + let scalars = (0 ... 0xFFFF).compactMap(Unicode.Scalar.init) + ["\u{10000}", "\u{1F600}", "\u{10FFFF}"] + for scalar in scalars { + let body: String + switch scalar { + case "\"": body = "\\\"" + case "\\": body = "\\\\" + default: body = JavaScriptText.lineBreakingEscape(scalar) ?? String(scalar) + } + if JavaScriptText.stringLiteral("x\(String(scalar))y") != "\"x\(body)y\"" { + mismatches.append(scalar.value) + } + } + #expect(mismatches.isEmpty, "first mismatches: \(mismatches.prefix(5))") + } + + @Test("A string literal holds no raw line terminator and reads back as the name through JSON") + func literalRoundTripsThroughJSON() throws { + let literal = JavaScriptText.stringLiteral(Self.mixedName) + #expect(!Self.containsRawLineTerminator(literal)) + let decoded = try JSONDecoder().decode(String.self, from: Data(literal.utf8)) + #expect(decoded.unicodeScalars.elementsEqual(Self.mixedName.unicodeScalars)) + } + + @Test("A string literal reads back as the name through JavaScriptCore's JSON.parse") + func literalRoundTripsThroughJavaScriptCore() throws { + let context = try #require(JSContext()) + let parse = try #require(context.evaluateScript("JSON.parse")) + for name in [Self.mixedName, "a\u{2028}b", "a\u{2029}b", "a\u{85}b", "plain"] { + let literal = JavaScriptText.stringLiteral(name) + let parsed = parse.call(withArguments: [literal]) + #expect(context.exception == nil) + #expect(parsed?.toString().unicodeScalars.elementsEqual(name.unicodeScalars) == true) + } + } + + @Test("A bridged string takes the same path as a native one") + func bridgedStringMatchesNative() { + let native = "a\u{2028}b\"c" + let bridged = NSString(string: native) as String + #expect(JavaScriptText.stringLiteral(bridged) == JavaScriptText.stringLiteral(native)) + #expect(JavaScriptText.stringLiteral(native) == "\"a\\u2028b\\\"c\"") + } + + @Test("A comment stays on one line whatever line terminator the text holds") + func commentHoldsNoLineTerminator() { + for terminator in Self.lineTerminators { + let comment = JavaScriptText.lineComment("Collection: a\(String(terminator))b") + #expect(!Self.containsRawLineTerminator(comment), "U+\(String(terminator.value, radix: 16))") + #expect(comment.hasPrefix("// Collection: a\\")) + } + } + + @Test("A comment keeps printable text as it is, quotes and a block-comment end included") + func commentKeepsPrintableText() { + #expect(JavaScriptText.lineComment("a */ \"b\" 'c' \\d tĂȘn") == "// a */ \"b\" 'c' \\d tĂȘn") + #expect(JavaScriptText.lineComment("") == "//") + } + + @Test("Only an ASCII identifier is plain") + func plainIdentifiers() { + for name in ["orders", "order_2", "_x", "A"] { + #expect(JavaScriptText.isPlainIdentifier(name), "\(name)") + } + for name in ["", "2025", "tĂȘn", "a b", "a;b", "a.b", "a$b", "a\u{0D4E}(\u{0D4E})", "a\u{2028}b"] { + #expect(!JavaScriptText.isPlainIdentifier(name), "\(name)") + } + } +} diff --git a/Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportUtilitiesTests.swift b/Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportUtilitiesTests.swift new file mode 100644 index 0000000000..1112d89d74 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportUtilitiesTests.swift @@ -0,0 +1,56 @@ +// +// PluginExportUtilitiesTests.swift +// TableProPluginKitTests +// + +import Foundation +import JavaScriptCore +import Testing + +@testable import TableProPluginKit + +struct PluginExportUtilitiesTests { + private static let separators = "a\u{85}b\u{2028}c\u{2029}d" + + @Test("NEL, the line separator and the paragraph separator are escaped") + func escapesLineSeparators() { + #expect(PluginExportUtilities.escapeJSONString(Self.separators) == "a\\u0085b\\u2028c\\u2029d") + } + + @Test("The escapes JSON export already wrote are unchanged") + func existingEscapesUnchanged() { + #expect(PluginExportUtilities.escapeJSONString("q\"b\\n\nr\rt\tb\u{08}f\u{0C}c\u{01}u\u{1F}") + == "q\\\"b\\\\n\\nr\\rt\\tb\\bf\\fc\\u0001u\\u001F") + } + + @Test("Characters that share a lead byte with a separator pass through as they are") + func neighboursOfSeparatorsUnchanged() { + let text = "\u{84}\u{86}\u{A0}\u{C0}\u{2027}\u{202A}\u{2030}\u{20AC}tĂȘn 東äșŹ 😀" + #expect(PluginExportUtilities.escapeJSONString(text) == text) + } + + @Test("A bridged string escapes the same way as a native one") + func bridgedStringMatchesNative() { + let bridged = NSString(string: Self.separators) as String + #expect(PluginExportUtilities.escapeJSONString(bridged) == "a\\u0085b\\u2028c\\u2029d") + } + + @Test("The escaped text reads back as the original through JSON and JavaScriptCore") + func roundTripsThroughJSONAndJavaScriptCore() throws { + let name = "x\"y\\z\n\u{0B}\u{0C}\(Self.separators)😀" + let literal = "\"\(PluginExportUtilities.escapeJSONString(name))\"" + let decoded = try JSONDecoder().decode(String.self, from: Data(literal.utf8)) + #expect(decoded.unicodeScalars.elementsEqual(name.unicodeScalars)) + + let context = try #require(JSContext()) + let parsed = context.evaluateScript("JSON.parse")?.call(withArguments: [literal]) + #expect(context.exception == nil) + #expect(parsed?.toString().unicodeScalars.elementsEqual(name.unicodeScalars) == true) + } + + @Test("A collection reached through getCollection has its separators escaped") + func accessorEscapesSeparators() { + #expect(MongoCollectionAccessor.expression(for: "a\u{2028}b") == "db.getCollection(\"a\\u2028b\")") + #expect(MongoCollectionAccessor.unescape("a\\u2028b") == "a\u{2028}b") + } +} diff --git a/Plugins/MQLExportPlugin/MQLCollectionDefinition.swift b/Plugins/MQLExportPlugin/MQLCollectionDefinition.swift new file mode 100644 index 0000000000..3d8cc79d3f --- /dev/null +++ b/Plugins/MQLExportPlugin/MQLCollectionDefinition.swift @@ -0,0 +1,141 @@ +// +// MQLCollectionDefinition.swift +// MQLExportPlugin +// + +import Foundation +import TableProJavaScriptText + +/// The index and validator statements an MQL export writes after a collection's documents, read +/// from the definition the MongoDB driver reports for it. +/// +/// That definition is shell text written around names the server chose, by whichever driver +/// version is installed, and copying it into the export copied whatever a name did to it. So it is +/// read instead, and only two shapes are written back: this collection's `createIndex` with +/// literal arguments, and a `collMod` that sets its validator. Each is written again from the +/// values read out of it. Anything else is left out, with a comment where it stood, and every +/// comment is written again on a line of its own. +/// +/// A statement is read only where the driver starts one, at the start of a line, and the +/// validator only as the first, where the driver writes it. A statement that fails to read is left +/// out up to the next line, not one token at a time: stepping through it found a statement a name +/// had spelled inside it and wrote that out as one of the driver's own. +enum MQLCollectionDefinition { + static let skippedStatementComment = "// Skipped a statement that is not an index or a validator" + + static func script(fromDDL ddl: String, collection: String) -> String { + let lexemes = MQLScriptLexer.lexemes(in: ddl) + guard let header = lexemes.firstIndex(where: isCollectionHeader) else { return "" } + var reader = MQLScriptReader(tokens: lexemes.map(\.token), index: header + 1) + var lines: [(text: String, followsBlankLine: Bool)] = [] + var isSkipping = false + var hasPassedStatement = false + + while let token = reader.current { + let lexeme = lexemes[reader.index] + if token == .punctuator(";") { + reader.advance() + } else if case .lineComment(let text) = token { + reader.advance() + let comment = JavaScriptText.lineComment(text.trimmingCharacters(in: .whitespaces)) + lines.append((comment, lexeme.followsBlankLine)) + isSkipping = false + } else if lexeme.followsLineBreak, + let statement = statement(&reader, collection: collection, readsValidator: !hasPassedStatement) { + lines.append((statement, lexeme.followsBlankLine)) + isSkipping = false + hasPassedStatement = true + } else { + skipToNextLine(&reader, lexemes: lexemes) + if !isSkipping { + lines.append((skippedStatementComment, lexeme.followsBlankLine)) + } + isSkipping = true + hasPassedStatement = true + } + } + + return lines.enumerated() + .map { offset, line in offset > 0 && line.followsBlankLine ? "\n" + line.text : line.text } + .joined(separator: "\n") + } + + private static func isCollectionHeader(_ lexeme: MQLScriptLexeme) -> Bool { + guard case .lineComment(let text) = lexeme.token else { return false } + return text.trimmingCharacters(in: .whitespaces).hasPrefix("Collection:") + } + + private static func statement( + _ reader: inout MQLScriptReader, + collection: String, + readsValidator: Bool + ) -> String? { + var attempt = reader + var text = createIndexStatement(&attempt, collection: collection) + if text == nil, readsValidator { + attempt = reader + text = validatorStatement(&attempt, collection: collection) + } + guard let text else { return nil } + reader = attempt + _ = reader.consume(";") + return text + } + + private static func skipToNextLine(_ reader: inout MQLScriptReader, lexemes: [MQLScriptLexeme]) { + repeat { + guard let token = reader.current else { return } + reader.advance() + if token == .punctuator(";") { return } + } while reader.index < lexemes.count && !lexemes[reader.index].followsLineBreak + } + + private static func createIndexStatement(_ reader: inout MQLScriptReader, collection: String) -> String? { + guard let name = accessedCollection(&reader), isSameName(name, collection), + reader.consume("."), reader.identifier() == "createIndex", reader.consume("("), + case .object(let keys)? = reader.value() else { + return nil + } + var arguments = [MQLScriptValue.object(keys)] + if reader.consume(","), reader.current != .punctuator(")") { + guard case .object(let options)? = reader.value() else { return nil } + arguments.append(.object(options)) + _ = reader.consume(",") + } + guard reader.consume(")") else { return nil } + let accessor = MQLExportHelpers.collectionAccessor(for: collection) + return "\(accessor).createIndex(\(arguments.map(\.compactText).joined(separator: ", ")));" + } + + private static func validatorStatement(_ reader: inout MQLScriptReader, collection: String) -> String? { + guard reader.identifier() == "db", reader.consume("."), reader.identifier() == "runCommand", + reader.consume("("), case .object(let command)? = reader.value(), reader.consume(")"), + command.count == 2, command[0].key == "collMod", command[1].key == "validator", + case .string(let name) = command[0].value, isSameName(name, collection), + case .object = command[1].value else { + return nil + } + let rebuilt = MQLScriptValue.object([MQLScriptMember(key: "collMod", value: .string(collection)), command[1]]) + return "db.runCommand(\(rebuilt.indentedText(depth: 0)));" + } + + /// `db.`, `db[""]` or `db.getCollection("")`, the three spellings a driver + /// has written a collection in. + private static func accessedCollection(_ reader: inout MQLScriptReader) -> String? { + guard reader.identifier() == "db" else { return nil } + if reader.consume("[") { + guard let name = reader.string(), reader.consume("]") else { return nil } + return name + } + guard reader.consume("."), let member = reader.identifier() else { return nil } + guard member == "getCollection", reader.current == .punctuator("(") else { return member } + guard reader.consume("("), let name = reader.string(), reader.consume(")") else { return nil } + return name + } + + /// Compared scalar by scalar: a server name is bytes, and `String` equality would take two + /// canonically equivalent names for the same collection. + private static func isSameName(_ name: String, _ collection: String) -> Bool { + name.unicodeScalars.elementsEqual(collection.unicodeScalars) + } +} diff --git a/Plugins/MQLExportPlugin/MQLExportHelpers.swift b/Plugins/MQLExportPlugin/MQLExportHelpers.swift index 857f95a8b1..71912cd839 100644 --- a/Plugins/MQLExportPlugin/MQLExportHelpers.swift +++ b/Plugins/MQLExportPlugin/MQLExportHelpers.swift @@ -4,12 +4,27 @@ // import Foundation +import TableProJavaScriptText import TableProNumberFormatting import TableProPluginKit enum MQLExportHelpers { + /// Spelled by the export's own rules rather than `MongoCollectionAccessor`'s, because this + /// plugin can be released for an app that shipped an older copy of that helper. static func collectionAccessor(for name: String) -> String { - MongoCollectionAccessor.expression(for: name) + guard JavaScriptText.isPlainIdentifier(name), !MongoCollectionAccessor.isShadowedByDatabaseMember(name) else { + return "db.getCollection(\(JavaScriptText.stringLiteral(name)))" + } + return "db.\(name)" + } + + static func headerComment(label: String, name: String) -> String { + JavaScriptText.lineComment("\(label): \(name)") + } + + static func documentLiteral(_ fields: [(name: String, value: String)]) -> String { + let members = fields.map { "\(JavaScriptText.stringLiteral($0.name)): \($0.value)" } + return " {\(members.joined(separator: ", "))}" } static func mqlBinaryValue(for data: Data, subtype: UInt8) -> String { @@ -23,13 +38,13 @@ enum MQLExportHelpers { case "ObjectId": let objectId = MongoDBObjectId(hex: value) guard objectId.isValid else { break } - return "ObjectId(\"\(PluginExportUtilities.escapeJSONString(value))\")" + return "ObjectId(\(JavaScriptText.stringLiteral(value)))" case "TIMESTAMP": guard isIso8601(value) else { break } - return "ISODate(\"\(PluginExportUtilities.escapeJSONString(value))\")" + return "ISODate(\(JavaScriptText.stringLiteral(value)))" case "DECIMAL": guard NumberText.isJSONNumberLiteral(value) else { break } - return "NumberDecimal(\"\(PluginExportUtilities.escapeJSONString(value))\")" + return "NumberDecimal(\(JavaScriptText.stringLiteral(value)))" default: break } @@ -61,10 +76,27 @@ enum MQLExportHelpers { (value.hasPrefix("[") && value.hasSuffix("]")) { if let data = value.data(using: .utf8), (try? JSONSerialization.jsonObject(with: data)) != nil { - return value + return escapingLineBreaks(inJSON: value) + } + } + return JavaScriptText.stringLiteral(value) + } + + /// `JSONSerialization` accepts U+2028, U+2029, DEL and the C1 controls raw inside a string, + /// and refuses every other character `lineBreakingEscape` answers for outside the whitespace + /// between tokens. So in JSON it accepted, each such character stands inside a string, where + /// its escape reads back as the same value. + private static func escapingLineBreaks(inJSON json: String) -> String { + var escaped = String.UnicodeScalarView() + for scalar in json.unicodeScalars { + if scalar == "\n" || scalar == "\r" || scalar == "\t" { + escaped.append(scalar) + } else if let escape = JavaScriptText.lineBreakingEscape(scalar) { + escaped.append(contentsOf: escape.unicodeScalars) + } else { + escaped.append(scalar) } - return "\"\(PluginExportUtilities.escapeJSONString(value))\"" } - return "\"\(PluginExportUtilities.escapeJSONString(value))\"" + return String(escaped) } } diff --git a/Plugins/MQLExportPlugin/MQLExportPlugin.swift b/Plugins/MQLExportPlugin/MQLExportPlugin.swift index ce67449432..a7854dcb81 100644 --- a/Plugins/MQLExportPlugin/MQLExportPlugin.swift +++ b/Plugins/MQLExportPlugin/MQLExportPlugin.swift @@ -70,7 +70,8 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let dbName = tables.first?.databaseName ?? "" if !dbName.isEmpty { - try fileHandle.write(contentsOf: "// Database: \(PluginExportUtilities.sanitizeForSQLComment(dbName))\n".toUTF8Data()) + let databaseHeader = MQLExportHelpers.headerComment(label: "Database", name: dbName) + try fileHandle.write(contentsOf: "\(databaseHeader)\n".toUTF8Data()) } try fileHandle.write(contentsOf: "\n".toUTF8Data()) @@ -86,8 +87,9 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let includeData = optionValue(table, at: 2) let collectionAccessor = MQLExportHelpers.collectionAccessor(for: table.name) + let collectionHeader = MQLExportHelpers.headerComment(label: "Collection", name: table.name) - try fileHandle.write(contentsOf: "// Collection: \(PluginExportUtilities.sanitizeForSQLComment(table.name))\n".toUTF8Data()) + try fileHandle.write(contentsOf: "\(collectionHeader)\n".toUTF8Data()) if includeDrop { try fileHandle.write(contentsOf: "\(collectionAccessor).drop();\n".toUTF8Data()) @@ -108,7 +110,7 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi columnTypeNames = header.columnTypeNames case .rows(let rows): for row in rows { - var fields: [String] = [] + var fields: [(name: String, value: String)] = [] for (colIndex, column) in columns.enumerated() { guard colIndex < row.count else { continue } let cell = row[colIndex] @@ -127,9 +129,9 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi for: value, columnTypeName: typeName ) } - fields.append("\"\(PluginExportUtilities.escapeJSONString(column))\": \(jsonValue)") + fields.append((name: column, value: jsonValue)) } - documentBatch.append(" {\(fields.joined(separator: ", "))}") + documentBatch.append(MQLExportHelpers.documentLiteral(fields)) if documentBatch.count >= batchSize { try writeMQLInsertMany( @@ -158,7 +160,6 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try await writeMQLIndexes( collection: table.name, databaseName: table.databaseName, - collectionAccessor: collectionAccessor, dataSource: dataSource, to: fileHandle ) @@ -199,7 +200,6 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func writeMQLIndexes( collection: String, databaseName: String, - collectionAccessor: String, dataSource: any PluginExportDataSource, to fileHandle: FileHandle ) async throws { @@ -207,30 +207,8 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi table: collection, databaseName: databaseName ) - - let lines = ddl.components(separatedBy: "\n") - var indexLines: [String] = [] - var foundHeader = false - - for line in lines { - if line.hasPrefix("// Collection:") { - foundHeader = true - continue - } - if foundHeader { - var processedLine = line - let escapedForDDL = collection.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") - let ddlAccessor = "db[\"\(escapedForDDL)\"]" - if processedLine.hasPrefix(ddlAccessor) { - processedLine = collectionAccessor + String(processedLine.dropFirst(ddlAccessor.count)) - } - indexLines.append(processedLine) - } - } - - let indexContent = indexLines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines) - if !indexContent.isEmpty { - try fileHandle.write(contentsOf: "\(indexContent)\n".toUTF8Data()) - } + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: collection) + guard !script.isEmpty else { return } + try fileHandle.write(contentsOf: "\(script)\n".toUTF8Data()) } } diff --git a/Plugins/MQLExportPlugin/MQLScriptLexer.swift b/Plugins/MQLExportPlugin/MQLScriptLexer.swift new file mode 100644 index 0000000000..519bc5c2d6 --- /dev/null +++ b/Plugins/MQLExportPlugin/MQLScriptLexer.swift @@ -0,0 +1,226 @@ +// +// MQLScriptLexer.swift +// MQLExportPlugin +// + +import Foundation + +enum MQLScriptToken: Equatable { + case identifier(String) + case string(String) + case number(String) + case punctuator(Unicode.Scalar) + case lineComment(String) + case unreadable +} + +struct MQLScriptLexeme: Equatable { + let token: MQLScriptToken + let followsLineBreak: Bool + let followsBlankLine: Bool +} + +/// Splits shell text into tokens where JavaScript would: a `//` comment and a string end at the +/// characters the language ends them at, not at the ones the text was expected to hold. +/// +/// Only what a collection definition needs is read. Anything else, a regular expression literal, +/// a template string or an operator, comes back as `unreadable`, and a string with an escape it +/// does not know or a raw line break in it does too. +struct MQLScriptLexer { + private static let punctuators: Set = ["{", "}", "[", "]", "(", ")", ",", ":", ";", ".", "-"] + private static let lineTerminators: Set = ["\n", "\r", "\u{2028}", "\u{2029}"] + private static let spaces: Set = [" ", "\t", "\u{0B}", "\u{0C}", "\u{A0}", "\u{FEFF}"] + + private let scalars: [Unicode.Scalar] + private var index = 0 + + private init(scalars: [Unicode.Scalar]) { + self.scalars = scalars + } + + static func lexemes(in source: String) -> [MQLScriptLexeme] { + var lexer = MQLScriptLexer(scalars: Array(source.unicodeScalars)) + var lexemes: [MQLScriptLexeme] = [] + while true { + let lineBreaks = lexer.skipTrivia() + guard let token = lexer.nextToken() else { return lexemes } + lexemes.append( + MQLScriptLexeme(token: token, followsLineBreak: lineBreaks > 0, followsBlankLine: lineBreaks > 1) + ) + } + } + + // MARK: - Trivia + + private mutating func skipTrivia() -> Int { + var lineBreaks = 0 + while let scalar = peek() { + if Self.lineTerminators.contains(scalar) { + index += scalar == "\r" && peek(1) == "\n" ? 2 : 1 + lineBreaks += 1 + } else if Self.spaces.contains(scalar) || scalar.properties.generalCategory == .spaceSeparator { + index += 1 + } else if scalar == "/", peek(1) == "*" { + lineBreaks += skipBlockComment() + } else { + break + } + } + return lineBreaks + } + + private mutating func skipBlockComment() -> Int { + index += 2 + var lineBreaks = 0 + while let scalar = peek() { + if scalar == "*", peek(1) == "/" { + index += 2 + return lineBreaks + } + if Self.lineTerminators.contains(scalar) { lineBreaks += 1 } + index += 1 + } + return lineBreaks + } + + // MARK: - Tokens + + private mutating func nextToken() -> MQLScriptToken? { + guard let scalar = peek() else { return nil } + if scalar == "/", peek(1) == "/" { return lineComment() } + if scalar == "\"" || scalar == "'" { return string(quotedBy: scalar) } + if Self.isDigit(scalar) || (scalar == "." && peek(1).map(Self.isDigit) == true) { return number() } + if Self.isIdentifierStart(scalar) { return identifier() } + index += 1 + return Self.punctuators.contains(scalar) ? .punctuator(scalar) : .unreadable + } + + private mutating func lineComment() -> MQLScriptToken { + index += 2 + var text = String.UnicodeScalarView() + while let scalar = peek(), !Self.lineTerminators.contains(scalar) { + text.append(scalar) + index += 1 + } + return .lineComment(String(text)) + } + + /// A raw line feed or carriage return ends the token unread and is left for the trivia, so the + /// next line is read on its own. A raw U+2028 or U+2029 is part of the string, as it is in + /// JavaScript since ES2019. + private mutating func string(quotedBy quote: Unicode.Scalar) -> MQLScriptToken { + index += 1 + var value = String.UnicodeScalarView() + var isReadable = true + while let scalar = peek(), scalar != "\n", scalar != "\r" { + index += 1 + if scalar == quote { + return isReadable ? .string(String(value)) : .unreadable + } + guard scalar == "\\" else { + value.append(scalar) + continue + } + if let escaped = escapedScalar() { + value.append(escaped) + } else { + isReadable = false + } + } + return .unreadable + } + + private mutating func escapedScalar() -> Unicode.Scalar? { + guard let scalar = peek() else { return nil } + index += 1 + switch scalar { + case "\"", "'", "\\", "/": return scalar + case "b": return "\u{08}" + case "f": return "\u{0C}" + case "n": return "\n" + case "r": return "\r" + case "t": return "\t" + case "v": return "\u{0B}" + case "0": return peek().map(Self.isDigit) == true ? nil : "\u{00}" + case "x": return hexScalar(digits: 2) + case "u": return unicodeEscape() + default: return nil + } + } + + private mutating func unicodeEscape() -> Unicode.Scalar? { + if peek() == "{" { + index += 1 + var digits = "" + while let scalar = peek(), scalar != "}", digits.count < 7 { + digits.unicodeScalars.append(scalar) + index += 1 + } + guard peek() == "}", !digits.isEmpty, let value = UInt32(digits, radix: 16) else { return nil } + index += 1 + return Unicode.Scalar(value) + } + guard let unit = hexValue(digits: 4) else { return nil } + guard (0xD800 ... 0xDBFF).contains(unit) else { return Unicode.Scalar(unit) } + guard peek() == "\\", peek(1) == "u" else { return nil } + index += 2 + guard let low = hexValue(digits: 4), (0xDC00 ... 0xDFFF).contains(low) else { return nil } + return Unicode.Scalar(0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)) + } + + private mutating func hexScalar(digits: Int) -> Unicode.Scalar? { + hexValue(digits: digits).flatMap(Unicode.Scalar.init) + } + + private mutating func hexValue(digits: Int) -> UInt32? { + guard index + digits <= scalars.count else { return nil } + var text = "" + text.unicodeScalars.append(contentsOf: scalars[index ..< index + digits]) + guard text.unicodeScalars.allSatisfy(\.properties.isASCIIHexDigit), let value = UInt32(text, radix: 16) else { + return nil + } + index += digits + return value + } + + /// Reads the whole run a number could be, letters included, and leaves the grammar to the + /// reader, so `0x1F` or `1n` is one token that fails there instead of two that might not. + private mutating func number() -> MQLScriptToken { + var text = String.UnicodeScalarView() + while let scalar = peek() { + let signsExponent = (scalar == "+" || scalar == "-") && (text.last == "e" || text.last == "E") + guard Self.isIdentifierPart(scalar) || scalar == "." || signsExponent else { break } + text.append(scalar) + index += 1 + } + return .number(String(text)) + } + + private mutating func identifier() -> MQLScriptToken { + var name = String.UnicodeScalarView() + while let scalar = peek(), Self.isIdentifierPart(scalar) { + name.append(scalar) + index += 1 + } + return .identifier(String(name)) + } + + // MARK: - Scalars + + private func peek(_ offset: Int = 0) -> Unicode.Scalar? { + let position = index + offset + return position < scalars.count ? scalars[position] : nil + } + + private static func isDigit(_ scalar: Unicode.Scalar) -> Bool { + ("0" ... "9").contains(scalar) + } + + private static func isIdentifierStart(_ scalar: Unicode.Scalar) -> Bool { + scalar == "$" || scalar == "_" || scalar.properties.isXIDStart + } + + private static func isIdentifierPart(_ scalar: Unicode.Scalar) -> Bool { + scalar == "$" || scalar == "_" || scalar == "\u{200C}" || scalar == "\u{200D}" || scalar.properties.isXIDContinue + } +} diff --git a/Plugins/MQLExportPlugin/MQLScriptValue.swift b/Plugins/MQLExportPlugin/MQLScriptValue.swift new file mode 100644 index 0000000000..efa2efe000 --- /dev/null +++ b/Plugins/MQLExportPlugin/MQLScriptValue.swift @@ -0,0 +1,177 @@ +// +// MQLScriptValue.swift +// MQLExportPlugin +// + +import Foundation +import TableProJavaScriptText +import TableProNumberFormatting + +/// A literal value read out of shell text, which the export writes again by its own rules rather +/// than copying the text it came from. +indirect enum MQLScriptValue: Equatable { + case string(String) + case number(String) + case boolean(Bool) + case null + case nonFinite(String) + case array([MQLScriptValue]) + case object([MQLScriptMember]) + case constructor(String, [MQLScriptValue]) + + /// The shell constructors a collection definition writes a typed value with. + static let constructors: Set = [ + "BinData", "Code", "Double", "ISODate", "MaxKey", "MinKey", "NumberDecimal", "NumberLong", "ObjectId", + "Timestamp" + ] + + var compactText: String { + switch self { + case .string(let value): + return JavaScriptText.stringLiteral(value) + case .number(let text), .nonFinite(let text): + return text + case .boolean(let value): + return value ? "true" : "false" + case .null: + return "null" + case .array(let elements): + return "[\(elements.map(\.compactText).joined(separator: ", "))]" + case .object(let members): + let pairs = members.map { "\(JavaScriptText.stringLiteral($0.key)): \($0.value.compactText)" } + return "{\(pairs.joined(separator: ", "))}" + case .constructor(let name, let arguments): + return "\(name)(\(arguments.map(\.compactText).joined(separator: ", ")))" + } + } + + /// One member or element per line, `depth` levels in, with a constructor call kept on one line. + func indentedText(depth: Int) -> String { + let inner = String(repeating: " ", count: depth + 1) + let outer = String(repeating: " ", count: depth) + switch self { + case .array(let elements) where !elements.isEmpty: + let lines = elements.map { inner + $0.indentedText(depth: depth + 1) } + return "[\n\(lines.joined(separator: ",\n"))\n\(outer)]" + case .object(let members) where !members.isEmpty: + let lines = members.map { + "\(inner)\(JavaScriptText.stringLiteral($0.key)): \($0.value.indentedText(depth: depth + 1))" + } + return "{\n\(lines.joined(separator: ",\n"))\n\(outer)}" + default: + return compactText + } + } +} + +struct MQLScriptMember: Equatable { + let key: String + let value: MQLScriptValue +} + +/// Reads statements and literal values from a token run, one construct at a time. +/// +/// Each read either returns what it matched and moves past it, or returns nil. A caller that needs +/// to try one shape and then another works on a copy, which is cheap because the tokens are shared. +struct MQLScriptReader { + private let tokens: [MQLScriptToken] + private(set) var index: Int + + init(tokens: [MQLScriptToken], index: Int = 0) { + self.tokens = tokens + self.index = index + } + + var current: MQLScriptToken? { + index < tokens.count ? tokens[index] : nil + } + + mutating func advance() { + index += 1 + } + + mutating func consume(_ punctuator: Unicode.Scalar) -> Bool { + guard current == .punctuator(punctuator) else { return false } + index += 1 + return true + } + + mutating func identifier() -> String? { + guard case .identifier(let name)? = current else { return nil } + index += 1 + return name + } + + mutating func string() -> String? { + guard case .string(let value)? = current else { return nil } + index += 1 + return value + } + + mutating func value() -> MQLScriptValue? { + guard let token = current else { return nil } + index += 1 + switch token { + case .string(let text): + return .string(text) + case .number(let text): + return NumberText.isJSONNumberLiteral(text) ? .number(text) : nil + case .punctuator("{"): + return members().map(MQLScriptValue.object) + case .punctuator("["): + return elements(closedBy: "]").map(MQLScriptValue.array) + case .punctuator("-"): + return negated() + case .identifier(let name): + return named(name) + default: + return nil + } + } + + private mutating func negated() -> MQLScriptValue? { + switch current { + case .number(let text)?: + index += 1 + let negative = "-" + text + return NumberText.isJSONNumberLiteral(negative) ? .number(negative) : nil + case .identifier("Infinity")?: + index += 1 + return .nonFinite("-Infinity") + default: + return nil + } + } + + private mutating func named(_ name: String) -> MQLScriptValue? { + switch name { + case "true": return .boolean(true) + case "false": return .boolean(false) + case "null": return .null + case "Infinity", "NaN": return .nonFinite(name) + default: + guard MQLScriptValue.constructors.contains(name), consume("(") else { return nil } + return elements(closedBy: ")").map { .constructor(name, $0) } + } + } + + private mutating func members() -> [MQLScriptMember]? { + var members: [MQLScriptMember] = [] + while !consume("}") { + guard let key = string() ?? identifier(), consume(":"), let value = value() else { return nil } + members.append(MQLScriptMember(key: key, value: value)) + guard consume(",") || current == .punctuator("}") else { return nil } + } + return members + } + + private mutating func elements(closedBy closer: Unicode.Scalar) -> [MQLScriptValue]? { + var elements: [MQLScriptValue] = [] + while !consume(closer) { + guard let element = value() else { return nil } + elements.append(element) + guard consume(",") || current == .punctuator(closer) else { return nil } + } + return elements + } +} diff --git a/Plugins/TableProPluginKit/PluginExportUtilities.swift b/Plugins/TableProPluginKit/PluginExportUtilities.swift index 222ec7ee62..a2ed3958f6 100644 --- a/Plugins/TableProPluginKit/PluginExportUtilities.swift +++ b/Plugins/TableProPluginKit/PluginExportUtilities.swift @@ -9,8 +9,20 @@ public enum PluginExportUtilities { public static func escapeJSONString(_ string: String) -> String { var utf8Result = [UInt8]() utf8Result.reserveCapacity(string.utf8.count) + let wasContiguous = string.utf8.withContiguousStorageIfAvailable { + appendJSONEscaped($0, to: &utf8Result) + } != nil + if !wasContiguous { + Array(string.utf8).withUnsafeBufferPointer { appendJSONEscaped($0, to: &utf8Result) } + } + return String(bytes: utf8Result, encoding: .utf8) ?? string + } - for byte in string.utf8 { + private static func appendJSONEscaped(_ bytes: UnsafeBufferPointer, to utf8Result: inout [UInt8]) { + var index = 0 + while index < bytes.count { + let byte = bytes[index] + index += 1 switch byte { case 0x22: // " utf8Result.append(0x5C) @@ -36,12 +48,17 @@ public enum PluginExportUtilities { case 0x00...0x1F: let hex = String(format: "\\u%04X", byte) utf8Result.append(contentsOf: hex.utf8) + case 0xC2 where index < bytes.count && bytes[index] == 0x85: + utf8Result.append(contentsOf: "\\u0085".utf8) + index += 1 + case 0xE2 where index + 1 < bytes.count && bytes[index] == 0x80 + && (bytes[index + 1] == 0xA8 || bytes[index + 1] == 0xA9): + utf8Result.append(contentsOf: (bytes[index + 1] == 0xA8 ? "\\u2028" : "\\u2029").utf8) + index += 2 default: utf8Result.append(byte) } } - - return String(bytes: utf8Result, encoding: .utf8) ?? string } @available(*, deprecated, message: "Use beginAtomicWrite(for:) for crash-safe writes") diff --git a/TablePro/Models/Query/QueryTab.swift b/TablePro/Models/Query/QueryTab.swift index a71a798d69..ebb4436e60 100644 --- a/TablePro/Models/Query/QueryTab.swift +++ b/TablePro/Models/Query/QueryTab.swift @@ -329,8 +329,7 @@ struct QueryTab: Identifiable, Equatable { switch PluginManager.shared.editorLanguage(for: databaseType) { case .javascript: - let escaped = tableName.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") - return "db[\"\(escaped)\"].find({}).limit(\(pageSize))" + return "\(MongoCollectionAccessor.expression(for: tableName)).find({}).limit(\(pageSize))" case .bash: return "SCAN 0 MATCH * COUNT \(pageSize)" default: diff --git a/TableProTests/Models/Query/QueryTabBaseQueryTests.swift b/TableProTests/Models/Query/QueryTabBaseQueryTests.swift index 1749e678c2..470c67de70 100644 --- a/TableProTests/Models/Query/QueryTabBaseQueryTests.swift +++ b/TableProTests/Models/Query/QueryTabBaseQueryTests.swift @@ -47,4 +47,15 @@ struct QueryTabBaseQueryTests { #expect(!query.uppercased().contains("SUBSTRING")) #expect(!query.hasSuffix(";")) } + + @Test("A JavaScript editor's own query reaches the collection through the shared accessor") + func javascriptQueryUsesSharedAccessor() throws { + let lineTerminators: Set = ["\n", "\r", "\u{0B}", "\u{0C}", "\u{85}", "\u{2028}", "\u{2029}"] + for name in ["users", "stats", "a\u{2028}b", "c\u{85}d", "e\"f\ng"] { + let query = try QueryTab.buildBaseTableQuery(tableName: name, databaseType: .mongodb) + #expect(query.hasPrefix("\(MongoCollectionAccessor.expression(for: name)).find({}).limit("), "\(name)") + #expect(!query.unicodeScalars.contains { lineTerminators.contains($0) }, "\(name)") + #expect(QuerySqlParser.extractTableName(from: query) == name, "\(name)") + } + } } diff --git a/TableProTests/Plugins/MQLCollectionDefinitionTests.swift b/TableProTests/Plugins/MQLCollectionDefinitionTests.swift new file mode 100644 index 0000000000..6442572cff --- /dev/null +++ b/TableProTests/Plugins/MQLCollectionDefinitionTests.swift @@ -0,0 +1,284 @@ +// +// MQLCollectionDefinitionTests.swift +// TableProTests +// + +import Foundation +import JavaScriptCore +import TableProJavaScriptText +import TableProNumberFormatting +import TableProPluginKit +import Testing + +struct MQLCollectionDefinitionTests { + private typealias IndexFixture = (name: String, key: [String: Any], options: [String]) + + private static let rawLineBreaks: Set = ["\r", "\u{0B}", "\u{0C}", "\u{85}", "\u{2028}", "\u{2029}"] + + /// The text the MongoDB driver on main writes for Show DDL, built the way it builds it: the + /// header and the `collMod` name are the name as it is, and the index name sits in quotes as it is. + private static func driverDDL( + collection: String, + capped: String? = nil, + validator: [String: Any]? = nil, + indexes: [IndexFixture] = [] + ) -> String { + var sections = ["// Collection: \(collection)"] + if let capped { sections.append(capped) } + if let validator, let json = NumberText.json(from: validator, prettyPrinted: true) { + sections.append("\n// Validator\ndb.runCommand({\n \"collMod\": \"\(collection)\",\n \"validator\": \(json)\n})") + } + if !indexes.isEmpty { + sections.append("\n// Indexes") + for index in indexes { + let key = NumberText.json(from: index.key, prettyPrinted: true) ?? "{}" + let options = (index.options + ["\"name\": \"\(index.name)\""]).joined(separator: ", ") + sections.append("\(MongoCollectionAccessor.expression(for: collection)).createIndex(\(key), {\(options)})") + } + } + return sections.joined(separator: "\n") + } + + /// Reading the export's own output back must find every statement again, so the output holds + /// only the shapes the reader accepts, and no line terminator but the line feeds between lines. + /// JavaScriptCore checks that it parses, without running it. + private static func expectAllowlistedShapes(_ script: String, collection: String) { + #expect(!script.unicodeScalars.contains { rawLineBreaks.contains($0) }) + let reread = MQLCollectionDefinition.script(fromDDL: "// Collection: x\n\(script)", collection: collection) + #expect(reread == script) + + let context = JSGlobalContextCreate(nil) + let source = JSStringCreateWithCFString(script as CFString) + defer { + JSStringRelease(source) + JSGlobalContextRelease(context) + } + #expect(JSCheckScriptSyntax(context, source, nil, 1, nil)) + } + + @Test("The driver's definition comes back as the same statements, written by the export's rules") + func driverDefinitionIsRewritten() { + let ddl = Self.driverDDL( + collection: "users", + capped: "// Capped: true, size: 4096", + validator: ["$jsonSchema": ["bsonType": "object", "required": ["email"]]], + indexes: [ + (name: "email_1", key: ["email": 1], options: ["\"unique\": true"]), + (name: "createdAt_1", key: ["createdAt": 1], options: ["\"expireAfterSeconds\": 3600", "\"sparse\": true"]) + ] + ) + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: "users") + #expect(script == """ + // Capped: true, size: 4096 + + // Validator + db.runCommand({ + "collMod": "users", + "validator": { + "$jsonSchema": { + "bsonType": "object", + "required": [ + "email" + ] + } + } + }); + + // Indexes + db.users.createIndex({"email": 1}, {"unique": true, "name": "email_1"}); + db.users.createIndex({"createdAt": 1}, {"expireAfterSeconds": 3600, "sparse": true, "name": "createdAt_1"}); + """) + Self.expectAllowlistedShapes(script, collection: "users") + } + + @Test("Shell constructors and libbson spacing read back as the same values") + func typedValuesAreRewritten() { + let ddl = """ + // Collection: events + // Time series: { "timeField" : "at" } + + // Validator + db.runCommand({ + "collMod": "events", + "validator": { + "at": { "$gte": ISODate("2026-01-01T00:00:00.000Z") }, + "n": { "$lt": NumberLong("9007199254740993") }, + "ratio": Double(1.0), + "d": NumberDecimal("1.10"), + "id": { "$ne": ObjectId("507f1f77bcf86cd799439011") }, + "b": { "$ne": BinData(4, "jNAD60olQySTMoj84toNGg==") }, + "t": { "$ne": Timestamp(1, 2) }, + "k": { "$gt": MinKey(), "$lt": MaxKey() }, + "x": { "$in": [ -1, 2.5e-7, -Infinity, NaN, null, false ] } + } + }) + + // Indexes + db.getCollection("events").createIndex({ "at" : -1, "n" : 1 }, { "name" : "at_-1_n_1", "partialFilterExpression" : { "n" : { "$gt" : NumberLong("5") } } }) + """ + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: "events") + #expect(script.contains("\"at\": {\n \"$gte\": ISODate(\"2026-01-01T00:00:00.000Z\")\n }")) + #expect(script.contains("\"n\": {\n \"$lt\": NumberLong(\"9007199254740993\")\n }")) + #expect(script.contains("\"ratio\": Double(1.0)")) + #expect(script.contains("\"b\": {\n \"$ne\": BinData(4, \"jNAD60olQySTMoj84toNGg==\")\n }")) + #expect(script.contains("\"$gt\": MinKey(),\n \"$lt\": MaxKey()")) + #expect(script.contains("-1,\n 2.5e-7,\n -Infinity,\n NaN,\n null,\n false")) + #expect(script.hasSuffix(""" + db.events.createIndex({"at": -1, "n": 1}, {"name": "at_-1_n_1", "partialFilterExpression": {"n": {"$gt": NumberLong("5")}}}); + """)) + #expect(!script.contains(MQLCollectionDefinition.skippedStatementComment)) + Self.expectAllowlistedShapes(script, collection: "events") + } + + @Test("A name holding a line terminator stays inside its comment and its string literals") + func lineTerminatorsInNamesStayInsideLiterals() throws { + let context = try #require(JSContext()) + let parse = try #require(context.evaluateScript("JSON.parse")) + for terminator: Unicode.Scalar in ["\u{85}", "\u{2028}", "\u{2029}"] { + let collection = "a\(String(terminator))b" + let indexName = "i\(String(terminator))j" + let ddl = """ + // Collection: \(JavaScriptText.lineComment(collection).dropFirst(3)) + + // Indexes + \(MongoCollectionAccessor.expression(for: collection)).createIndex({"k\(String(terminator))": 1}, {"name": \(JavaScriptText.stringLiteral(indexName))}) + """ + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: collection) + let keyName = "k\(String(terminator))" + #expect(script.contains("db.getCollection(\(JavaScriptText.stringLiteral(collection))).createIndex(")) + for name in [collection, indexName, keyName] { + let literal = JavaScriptText.stringLiteral(name) + #expect(script.contains(literal)) + let parsed = parse.call(withArguments: [literal])?.toString() + #expect(parsed?.unicodeScalars.elementsEqual(name.unicodeScalars) == true) + } + Self.expectAllowlistedShapes(script, collection: collection) + } + } + + @Test("A header split by a line feed in the name leaves its second line out") + func headerSplitByLineFeedIsLeftOut() { + let collection = "a\nb" + let ddl = Self.driverDDL(collection: collection, indexes: [(name: "k_1", key: ["k": 1], options: [])]) + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: collection) + #expect(script == """ + \(MQLCollectionDefinition.skippedStatementComment) + + // Indexes + db.getCollection("a\\nb").createIndex({"k": 1}, {"name": "k_1"}); + """) + Self.expectAllowlistedShapes(script, collection: collection) + } + + @Test("A validator whose collection name was written unescaped is left out, and the indexes kept") + func unescapedCollModNameIsLeftOut() { + let collection = "a\"b" + let ddl = Self.driverDDL( + collection: collection, + validator: ["$jsonSchema": ["bsonType": "object"]], + indexes: [(name: "k_1", key: ["k": 1], options: [])] + ) + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: collection) + #expect(!script.contains("runCommand")) + #expect(script.contains(MQLCollectionDefinition.skippedStatementComment)) + #expect(script.hasSuffix("db.getCollection(\"a\\\"b\").createIndex({\"k\": 1}, {\"name\": \"k_1\"});")) + Self.expectAllowlistedShapes(script, collection: collection) + } + + @Test("An index whose name was written unescaped is left out") + func unescapedIndexNameIsLeftOut() { + let ddl = Self.driverDDL( + collection: "users", + indexes: [(name: "a\"b", key: ["k": 1], options: []), (name: "email_1", key: ["email": 1], options: [])] + ) + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: "users") + #expect(script == """ + // Indexes + \(MQLCollectionDefinition.skippedStatementComment) + db.users.createIndex({"email": 1}, {"name": "email_1"}); + """) + Self.expectAllowlistedShapes(script, collection: "users") + } + + @Test("A statement spelled inside an index name that fails to read is left out with it") + func statementInsideAFailedStatementIsLeftOut() { + let name = #"x" @ db.runCommand({"collMod": "users", "validator": {"$expr": false}}) @ ""# + let ddl = Self.driverDDL( + collection: "users", + indexes: [(name: name, key: ["k": 1], options: []), (name: "email_1", key: ["email": 1], options: [])] + ) + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: "users") + #expect(script == """ + // Indexes + \(MQLCollectionDefinition.skippedStatementComment) + db.users.createIndex({"email": 1}, {"name": "email_1"}); + """) + Self.expectAllowlistedShapes(script, collection: "users") + } + + @Test("A validator anywhere but first, where the driver writes it, is left out") + func validatorAfterAnIndexIsLeftOut() { + let name = "x\"})\ndb.runCommand({\"collMod\": \"users\", \"validator\": {\"$expr\": false}})\n//" + let ddl = Self.driverDDL(collection: "users", indexes: [(name: name, key: ["k": 1], options: [])]) + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: "users") + #expect(script == """ + // Indexes + db.users.createIndex({"k": 1}, {"name": "x"}); + \(MQLCollectionDefinition.skippedStatementComment) + // "}) + """) + Self.expectAllowlistedShapes(script, collection: "users") + } + + @Test("A statement that does not start its own line is left out") + func statementSharingALineIsLeftOut() { + let name = #"x"}); db.users.createIndex({"evil": 1}, {"unique": true}); //"# + let ddl = Self.driverDDL(collection: "users", indexes: [(name: name, key: ["k": 1], options: [])]) + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: "users") + #expect(script == """ + // Indexes + db.users.createIndex({"k": 1}, {"name": "x"}); + \(MQLCollectionDefinition.skippedStatementComment) + // "}) + """) + Self.expectAllowlistedShapes(script, collection: "users") + } + + @Test("Statements for another collection or of any other kind are left out") + func otherStatementsAreLeftOut() { + let ddl = """ + // Collection: users + db.orders.createIndex({"k": 1}, {"name": "k_1"}) + db.users.drop() + db.users.createIndex({"k": 1}, {"name": "k_1"}) + db.runCommand({"collMod": "orders", "validator": {}}) + db.runCommand({"collMod": "users", "validator": {}, "validationLevel": "off"}) + """ + let script = MQLCollectionDefinition.script(fromDDL: ddl, collection: "users") + #expect(script == """ + \(MQLCollectionDefinition.skippedStatementComment) + db.users.createIndex({"k": 1}, {"name": "k_1"}); + \(MQLCollectionDefinition.skippedStatementComment) + """) + } + + @Test("Every spelling of this collection's accessor is read, and written back one way") + func accessorSpellingsAreRead() { + for accessor in ["db.users", "db[\"users\"]", "db['users']", "db.getCollection(\"users\")"] { + let ddl = "// Collection: users\n\(accessor).createIndex({\"k\": 1})" + #expect(MQLCollectionDefinition.script(fromDDL: ddl, collection: "users") == "db.users.createIndex({\"k\": 1});") + } + } + + @Test("A definition with no collection header, such as a view's, writes nothing") + func viewDefinitionWritesNothing() { + let ddl = "// View: recent\ndb.createView(\"recent\", \"orders\", [ ])" + #expect(MQLCollectionDefinition.script(fromDDL: ddl, collection: "recent").isEmpty) + #expect(MQLCollectionDefinition.script(fromDDL: "", collection: "recent").isEmpty) + } + + @Test("A definition with nothing after its header writes nothing") + func headerOnlyWritesNothing() { + #expect(MQLCollectionDefinition.script(fromDDL: "// Collection: users", collection: "users").isEmpty) + } +} diff --git a/TableProTests/Plugins/MQLExportHelpersTests.swift b/TableProTests/Plugins/MQLExportHelpersTests.swift index 14a27a93e4..fb1594fdac 100644 --- a/TableProTests/Plugins/MQLExportHelpersTests.swift +++ b/TableProTests/Plugins/MQLExportHelpersTests.swift @@ -4,11 +4,15 @@ // import Foundation +@testable import TablePro import TableProPluginKit import Testing struct MQLExportHelpersTests { private static let uuid = "8cd003eb-4a25-4324-9332-88fce2da0d1a" + private static let lineTerminators: [Unicode.Scalar] = [ + "\n", "\r", "\u{0B}", "\u{0C}", "\u{85}", "\u{2028}", "\u{2029}" + ] /// The dump is a mongosh script, so a value has to be a constructor call. mongosh reads /// `{"$binary": ...}` as a plain object literal and would insert a subdocument. @@ -129,4 +133,54 @@ struct MQLExportHelpersTests { #expect(MQLExportHelpers.collectionAccessor(for: "my.data") == "db.getCollection(\"my.data\")") #expect(MQLExportHelpers.collectionAccessor(for: "2024") == "db.getCollection(\"2024\")") } + + @Test("The export spells a collection the way the shared accessor does") + func accessorAgreesWithSharedAccessor() { + let names = ["users", "order_2", "stats", "__proto", "my.data", "2024", "tĂȘn", "a\u{0D4E}(\u{0D4E})", "a\"b", "a\u{2028}b", ""] + for name in names { + #expect(MQLExportHelpers.collectionAccessor(for: name) == MongoCollectionAccessor.expression(for: name), "\(name)") + } + #expect(MQLExportHelpers.collectionAccessor(for: "a\u{2028}b") == "db.getCollection(\"a\\u2028b\")") + } + + @Test("A header comment stays on one line whatever line terminator the name holds") + func headerCommentStaysOnOneLine() { + for terminator in Self.lineTerminators { + let header = MQLExportHelpers.headerComment(label: "Collection", name: "a\(String(terminator))b") + #expect(!header.unicodeScalars.contains { Self.lineTerminators.contains($0) }, "U+\(terminator.value)") + #expect(header.hasPrefix("// Collection: a\\")) + let statements = JavaScriptStatementScanner.executableStatements(in: "x = 1;" + header) + #expect(statements.map(\.trimmed) == ["x = 1;"], "U+\(terminator.value)") + } + #expect(MQLExportHelpers.headerComment(label: "Database", name: "shop */ 'x'") == "// Database: shop */ 'x'") + } + + @Test("A document's field names and string values read back as they were") + func documentLiteralRoundTrips() throws { + let names = ["a\u{2028}b", "c\u{85}d", "e\u{2029}f", "g\"h", "i\nj", "k\u{0B}l"] + let fields = names.map { (name: $0, value: MQLExportHelpers.mqlJsonValue(for: $0)) } + let literal = MQLExportHelpers.documentLiteral(fields) + #expect(!literal.unicodeScalars.contains { Self.lineTerminators.contains($0) }) + let document = try #require(try JSONSerialization.jsonObject(with: Data(literal.utf8)) as? [String: String]) + for name in names { + #expect(document[name] == name) + } + } + + @Test("A typed value's string argument escapes line separators too") + func typedValueEscapesSeparators() { + #expect(MQLExportHelpers.mqlTextValue(for: "a\u{2028}b", columnTypeName: "VARCHAR") == "\"a\\u2028b\"") + #expect(MQLExportHelpers.mqlJsonValue(for: "x\u{85}") == "\"x\\u0085\"") + } + + @Test("A nested document keeps its text, with separators inside its strings escaped") + func nestedDocumentEscapesSeparators() throws { + let nested = "{\"k\": \"a\u{2028}b\", \"n\": [1, \"c\u{85}d\u{7F}\"]}" + let value = MQLExportHelpers.mqlJsonValue(for: nested) + #expect(value == "{\"k\": \"a\\u2028b\", \"n\": [1, \"c\\u0085d\\u007f\"]}") + let original = try #require(try JSONSerialization.jsonObject(with: Data(nested.utf8)) as? NSDictionary) + let written = try #require(try JSONSerialization.jsonObject(with: Data(value.utf8)) as? NSDictionary) + #expect(original == written) + #expect(MQLExportHelpers.mqlJsonValue(for: "{\n \"k\": 1\n}") == "{\n \"k\": 1\n}") + } } diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 60de8c31c6..0817ca5eb0 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -173,6 +173,8 @@ As SQL, a results export writes `INSERT` statements only. A result set is the ou ``` Top-level values keep their type: `ObjectId`, `ISODate`, `BinData` with its real subtype. A typed value nested inside a subdocument or array is written as a string, so re-importing gives you a string where the original held an ObjectId, date, or binary. Export those collections as JSON, or use `mongodump`. + + **Indexes** writes each index as a `createIndex()` call and the collection's validator as a `collMod` command. Any other statement in the collection's DDL is left out, and the line `// Skipped a statement that is not an index or a validator` marks where it stood. | Option | Default | diff --git a/project.yml b/project.yml index ecd82d75d7..c79ba3398c 100644 --- a/project.yml +++ b/project.yml @@ -503,7 +503,10 @@ targets: - Plugins/MSSQLDriverPlugin/MSSQLLoginParameters.swift - Plugins/MSSQLDriverPlugin/MSSQLSSLMapping.swift - Plugins/MSSQLDriverPlugin/MSSQLSessionTransaction.swift + - Plugins/MQLExportPlugin/MQLCollectionDefinition.swift - Plugins/MQLExportPlugin/MQLExportHelpers.swift + - Plugins/MQLExportPlugin/MQLScriptLexer.swift + - Plugins/MQLExportPlugin/MQLScriptValue.swift - Plugins/MongoDBDriverPlugin/BsonDocumentFlattener.swift - Plugins/MongoDBDriverPlugin/MongoDBDecimal128.swift - Plugins/MongoDBDriverPlugin/MongoDBAuthSourceResolver.swift @@ -753,7 +756,7 @@ targets: dependencies: - target: TablePro - package: TableProCore - products: [TableProConnectionLibrary, TableProDocumentPath, TableProGeometry, TableProLogRedaction, TableProMSSQLCore, TableProNumberFormatting, TableProSQLGrammar, TableProSSHTransport, TableProTabular, TableProTabularIO, TableProWeaviateCore] + products: [TableProConnectionLibrary, TableProDocumentPath, TableProGeometry, TableProJavaScriptText, TableProLogRedaction, TableProMSSQLCore, TableProNumberFormatting, TableProSQLGrammar, TableProSSHTransport, TableProTabular, TableProTabularIO, TableProWeaviateCore] # The Kafka integration suite drives the real driver, so the test target links what # the plugin target links: zstd for decompression and NIO for the transport. - package: zstd @@ -1027,11 +1030,17 @@ targets: folder: MQLExportPlugin principalClass: MQLExportPlugin dependencies: + - package: TableProCore + product: TableProJavaScriptText - package: TableProCore product: TableProNumberFormatting settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.MQLExportPlugin + # Above plugin-mql-v1.0.7, the newest copy published to the registry. `selectWinners` + # compares versions numerically, so at the template's 1.0 a retained user copy would keep + # writing server-provided names into exports unescaped. + MARKETING_VERSION: "1.0.8" SQLImport: templates: [DriverPlugin]