diff --git a/CHANGELOG.md b/CHANGELOG.md index 68de66babc..b13aa3dcfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -506,6 +506,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Previous run's column headings left over an empty grid under the error of a script that failed partway. - Row numbers of a longer previous result left beside the rows of a shorter one. - Run executing an old sorted query after a column header was clicked while a query ran. +- SQL import into SQL Server sending `GO` lines to the server and splitting batches at each semicolon. +- SQL Server dumps failing on their first view, routine or trigger when restored with sqlcmd or imported. ### Security diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBatchSeparator.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBatchSeparator.swift index 7755445064..5a181e56d3 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBatchSeparator.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBatchSeparator.swift @@ -39,11 +39,26 @@ public struct SQLBatchSeparator: Sendable, Equatable { in text: NSString, length: Int, grammar: SQLLexicalGrammar + ) -> SQLBatchSeparator? { + guard startsLine(text, at: offset) else { return nil } + return line(startingAt: offset, in: text, length: length, grammar: grammar) + } + + /// The separator whose `GO` starts at `offset`, for a reader that has already seen that nothing but spaces and + /// tabs stand before it on its line, such as one reading a file a chunk at a time whose buffer no longer holds + /// the start of the line. + /// + /// `text` has to hold the whole line, its line break included, unless the line ends the text: the rest of the + /// line is what decides, and a line cut short at `length` reads as ending there. + public static func line( + startingAt offset: Int, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar ) -> SQLBatchSeparator? { guard offset + 1 < length, isG(text.character(at: offset)), - isO(text.character(at: offset + 1)), - startsLine(text, at: offset) + isO(text.character(at: offset + 1)) else { return nil } diff --git a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBatchSeparatorTests.swift b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBatchSeparatorTests.swift index 1914dead71..418dddb1b0 100644 --- a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBatchSeparatorTests.swift +++ b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBatchSeparatorTests.swift @@ -83,6 +83,30 @@ struct SQLBatchSeparatorTests { #expect(separators(text).isEmpty) } + @Test( + "A reader that tracked the line start reads each line as the scanner does", + arguments: [ + "GO", "go", "GO ", "GO -- batch one", "GO--glued", "GO 2 -- twice", "GO\t7", "GO 05", "GO 2147483647", + "GO;", "GO 0", "GO -1", "GO x", "GO 2147483648", "GOTO done", "go_table", "GO5", "GO /* c */", "GO 2 3", + ] + ) + func lineStartingAtAgreesWithTheScanner(line: String) { + let text = "SELECT 1\n\(line)\nSELECT 2" + let lineStart = 9 + let buffer = "\(line)\nSELECT 2" as NSString + let read = SQLBatchSeparator.line(startingAt: 0, in: buffer, length: buffer.length, grammar: Self.sqlServer) + let scanned = separators(text).first + #expect(read?.repeatCount == scanned?.repeatCount) + #expect(read.map { NSRange(location: $0.range.location + lineStart, length: $0.range.length) } == scanned?.range) + } + + @Test("The caller vouches for the line start, so text before the GO is not read") + func lineStartingAtTrustsTheCaller() { + let buffer = " GO 3\nSELECT 2" as NSString + let read = SQLBatchSeparator.line(startingAt: 2, in: buffer, length: buffer.length, grammar: Self.sqlServer) + #expect(read == SQLBatchSeparator(range: NSRange(location: 2, length: 4), repeatCount: 3)) + } + @Test("GO after code on the same line is not a separator") func goMustStartTheLine() { #expect(separators("SELECT 1 GO\nSELECT 2").isEmpty) diff --git a/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/Plugins/SQLExportPlugin/SQLExportPlugin.swift index 5b2b74f342..137d9b2553 100644 --- a/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -88,6 +88,13 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi /// says so rather than shipping a dump whose three phases disagree. var exportSpansContainers = false + /// What ends every statement of the dump: a line break, and on an engine whose client runs a + /// script in batches cut at `GO` lines, that line too. SQL Server refuses a view, a routine or a + /// trigger that is not the first statement of its batch, and a routine's body runs to the end + /// of its batch, so each statement is written as a batch of its own, the way SQL Server + /// Management Studio writes a script. + private var statementEnd = "\n" + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportPlugin") required init() { loadSettings() } @@ -134,6 +141,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi exportSpansContainers = false tablesUnorderedByCycle = [] emittedSequenceNames = [] + statementEnd = dataSource.lexicalFeatures.contains(.batchSeparatorLines) ? "\nGO\n" : "\n" /// Read once, because `PluginManager` hands every window the same plugin instance and a /// second window's options pane can write `settings` while this export is still running. A @@ -470,7 +478,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi guard !dropTargets.isEmpty else { return } for object in dropTargets { guard let statement = dropStatement(for: object, dataSource: dataSource) else { continue } - try writer.write("\(statement)\n") + try writer.write(statement + statementEnd) } try writer.write("\n") } @@ -527,9 +535,9 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let quotedName = "\"\(seq.name.replacingOccurrences(of: "\"", with: "\"\""))\"" if optionValue(table, at: 1) { try writer.write( - "DROP SEQUENCE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\n") + "DROP SEQUENCE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\(statementEnd)") } - try writer.write("\(seq.ddl)\n\n") + try writer.write("\(seq.ddl)\(statementEnd)\n") } } catch { let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(table.name) @@ -547,10 +555,11 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let quotedName = "\"\(enumType.name.replacingOccurrences(of: "\"", with: "\"\""))\"" if optionValue(table, at: 1) { try writer.write( - "DROP TYPE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\n") + "DROP TYPE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\(statementEnd)") } let quotedLabels = enumType.labels.map { "'\(dataSource.escapeStringLiteral($0))'" } - try writer.write("CREATE TYPE \(quotedName) AS ENUM (\(quotedLabels.joined(separator: ", ")));\n\n") + let labels = quotedLabels.joined(separator: ", ") + try writer.write("CREATE TYPE \(quotedName) AS ENUM (\(labels));\(statementEnd)\n") } } catch { let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(table.name) @@ -588,8 +597,8 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - try writer.write(dataSource.scriptText(for: ddl)) - try writer.write("\n\n") + try writer.write(dataSource.scriptText(for: ddl) + statementEnd) + try writer.write("\n") } catch { ddlFailures.append(sanitizedName) let ddlWarning = "Warning: failed to fetch DDL for table \(sanitizedName): \(error)" @@ -631,7 +640,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi table: object.name, databaseName: object.databaseName) guard !statements.isEmpty else { return } for statement in statements { - try writer.write("\(dataSource.scriptText(for: statement))\n") + try writer.write(dataSource.scriptText(for: statement) + statementEnd) } try writer.write("\n") } catch { @@ -674,8 +683,8 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - try writer.write(dataSource.scriptText(for: ddl)) - try writer.write("\n\n") + try writer.write(dataSource.scriptText(for: ddl) + statementEnd) + try writer.write("\n") } catch { ddlFailures.append(sanitizedName) Self.logger.warning("Failed to fetch DDL for \(sanitizedName): \(error)") @@ -708,7 +717,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi principal: principal.name, host: principal.identity) guard !statements.isEmpty else { continue } for statement in statements { - try writer.write("\(dataSource.scriptText(for: statement))\n") + try writer.write(dataSource.scriptText(for: statement) + statementEnd) } } catch { let sanitized = PluginExportUtilities.sanitizeForSQLComment(principal.name) @@ -778,7 +787,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let grouped = groupForeignKeysByConstraint(fks) for group in grouped { let alter = renderAddConstraintFK(table: table, group: group, dataSource: dataSource) - try writer.write("\(alter)\n") + try writer.write(alter + statementEnd) emittedAnything = true } } @@ -793,7 +802,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi for column in columns where column.isIdentity { let setval = renderIdentitySetval( table: table, columnName: column.name, dataSource: dataSource) - try writer.write("\(setval)\n") + try writer.write(setval + statementEnd) emittedAnything = true } } @@ -829,7 +838,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let statements = try await dataSource.fetchIndexDDL( table: object.name, databaseName: object.databaseName) for statement in statements { - try writer.write("\(dataSource.scriptText(for: statement))\n") + try writer.write(dataSource.scriptText(for: statement) + statementEnd) emittedAnything = true } } catch { @@ -950,7 +959,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let needsIdentityInsert = dataSource.databaseTypeId == "SQL Server" && columnInfo.contains(where: \.isIdentity) let identityInsert = needsIdentityInsert - ? SQLExportSessionScope.identityInsert(tableRef: tableRef) + ? SQLExportSessionScope.identityInsert(tableRef: tableRef, statementEnd: statementEnd) : nil if !table.rowScope.isUnrestricted { @@ -1093,7 +1102,8 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let accumulator = SQLExportStatementAccumulator( prefix: rendered.prefix, suffix: rendered.suffix, - budget: statementBudget(for: dataSource.databaseTypeId, options: options)) + budget: statementBudget(for: dataSource.databaseTypeId, options: options), + terminator: ";\(statementEnd)\n") return (accumulator, rendered.warning) } diff --git a/Plugins/SQLExportPlugin/SQLExportSessionScope.swift b/Plugins/SQLExportPlugin/SQLExportSessionScope.swift index 64f81b9d2b..a0d5dc52e6 100644 --- a/Plugins/SQLExportPlugin/SQLExportSessionScope.swift +++ b/Plugins/SQLExportPlugin/SQLExportSessionScope.swift @@ -21,9 +21,12 @@ internal struct SQLExportSessionScope: Equatable { /// plain statements, one `ON` per table and one `OFF` after the stream, a rotation between them /// left part N+1 answering every row with "Cannot insert explicit value for identity column" /// while the export reported success (#2533). - internal static func identityInsert(tableRef: String) -> SQLExportSessionScope { + /// + /// `statementEnd` is what ends every statement of the dump, a `GO` line included where the dump + /// carries one, so the pair are statements like any other. + internal static func identityInsert(tableRef: String, statementEnd: String) -> SQLExportSessionScope { SQLExportSessionScope( - opener: "SET IDENTITY_INSERT \(tableRef) ON;\n", - closer: "SET IDENTITY_INSERT \(tableRef) OFF;\n") + opener: "SET IDENTITY_INSERT \(tableRef) ON;\(statementEnd)", + closer: "SET IDENTITY_INSERT \(tableRef) OFF;\(statementEnd)") } } diff --git a/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift b/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift index 92d80f668b..06c77f5d07 100644 --- a/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift +++ b/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift @@ -70,17 +70,17 @@ internal struct SQLExportStatementTally: Equatable { internal final class SQLExportStatementAccumulator { /// What separates two rows, and what terminates a statement. Both are counted as the UTF-8 bytes /// they are, because the file is written as UTF-8 whatever encoding its prologue declares to the - /// server. + /// server. A terminator can carry a `GO` line, which ends each statement's batch on SQL Server. private static let rowSeparator = ",\n" - private static let terminator = ";\n\n" private static let rowSeparatorBytes = 2 - private static let terminatorBytes = 3 private let prefix: String private let suffix: String + private let terminator: String private let budget: SQLExportStatementBudget private let prefixBytes: Int private let suffixBytes: Int + private let terminatorBytes: Int private var rows: [String] = [] private var bodyBytes = 0 @@ -90,12 +90,19 @@ internal final class SQLExportStatementAccumulator { internal private(set) var oversizedRowCount = 0 internal private(set) var statementCount = 0 - internal init(prefix: String, suffix: String, budget: SQLExportStatementBudget) { + internal init( + prefix: String, + suffix: String, + budget: SQLExportStatementBudget, + terminator: String = ";\n\n" + ) { self.prefix = prefix self.suffix = suffix + self.terminator = terminator self.budget = budget prefixBytes = prefix.utf8.count suffixBytes = suffix.utf8.count + terminatorBytes = terminator.utf8.count } /// What this accumulator wrote, against the limit it was built with rather than whatever the @@ -112,7 +119,7 @@ internal final class SQLExportStatementAccumulator { /// trailing clause is not small: `ON DUPLICATE KEY UPDATE` over a wide table, or a PostgreSQL /// `ON CONFLICT ... DO UPDATE SET`, runs to hundreds of bytes that a budget counting only values /// would spend twice. - private var envelopeBytes: Int { prefixBytes + suffixBytes + Self.terminatorBytes } + private var envelopeBytes: Int { prefixBytes + suffixBytes + terminatorBytes } /// Takes one rendered row and hands back the statement it closed, if it closed one. internal func append(_ renderedRow: String) -> String? { @@ -142,7 +149,7 @@ internal final class SQLExportStatementAccumulator { /// Closes whatever is held, and answers nil when nothing is. internal func finish() -> String? { guard !rows.isEmpty else { return nil } - let statement = prefix + rows.joined(separator: Self.rowSeparator) + suffix + Self.terminator + let statement = prefix + rows.joined(separator: Self.rowSeparator) + suffix + terminator let statementBytes = envelopeBytes + bodyBytes if statementBytes > largestStatementBytes { largestStatementBytes = statementBytes diff --git a/Plugins/SQLImportPlugin/SQLImportPlugin.swift b/Plugins/SQLImportPlugin/SQLImportPlugin.swift index f3e502458e..ed5c6928c1 100644 --- a/Plugins/SQLImportPlugin/SQLImportPlugin.swift +++ b/Plugins/SQLImportPlugin/SQLImportPlugin.swift @@ -73,7 +73,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi try progress.checkCancellation() do { - try await sink.execute(statement: statement) + try await sink.execute(statement: statement, line: lineNumber) executedCount += 1 progress.incrementStatement() } catch { diff --git a/Plugins/TableProPluginKit/PluginExportDataSource.swift b/Plugins/TableProPluginKit/PluginExportDataSource.swift index 3c6b2d5d2a..2dc7424985 100644 --- a/Plugins/TableProPluginKit/PluginExportDataSource.swift +++ b/Plugins/TableProPluginKit/PluginExportDataSource.swift @@ -40,6 +40,12 @@ public protocol PluginExportDataSource: AnyObject, Sendable { /// does not already end in one. func scriptText(for ddl: String) -> String + /// How this engine's own client reads a script, so a dump is written the way it is read back. SQL Server's + /// client ends a batch at a line holding only `GO` (``SQLLexicalFeatures/batchSeparatorLines``) and refuses a + /// view, a routine or a trigger that is not the first statement of its batch. Empty by default, which writes a + /// script of `;`-terminated statements. + var lexicalFeatures: SQLLexicalFeatures { get } + /// The GRANT statements that recreate one principal's privileges, rendered by the engine's own /// grant builder. `host` is the MySQL-style host part, which is what separates two principals /// that share a name. Empty on an engine with no principal management. @@ -86,6 +92,8 @@ public extension PluginExportDataSource { ddl.hasSuffix(";") ? ddl : ddl + ";" } + var lexicalFeatures: SQLLexicalFeatures { [] } + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { [] } var supportsCascadeDrop: Bool { false } diff --git a/Plugins/TableProPluginKit/PluginImportDataSink.swift b/Plugins/TableProPluginKit/PluginImportDataSink.swift index ef983d39c8..21d25f4d95 100644 --- a/Plugins/TableProPluginKit/PluginImportDataSink.swift +++ b/Plugins/TableProPluginKit/PluginImportDataSink.swift @@ -9,6 +9,12 @@ public protocol PluginImportDataSink: AnyObject, Sendable { var databaseTypeId: String { get } var targetTable: String? { get } func execute(statement: String) async throws + + /// Runs one statement read from a file that starts on the file's `line`, so an error the database places inside + /// the statement can be placed in the file. On SQL Server the statement is a whole batch, and the server counts + /// its lines from the batch's first. + func execute(statement: String, line: Int) async throws + func insertRow(_ values: [String: PluginCellValue]) async throws func insertRows(_ rows: [[String: PluginCellValue]]) async throws func deleteAllRowsFromTargetTable() async throws @@ -21,6 +27,9 @@ public protocol PluginImportDataSink: AnyObject, Sendable { public extension PluginImportDataSink { var targetTable: String? { nil } + func execute(statement: String, line: Int) async throws { + try await execute(statement: statement) + } func insertRow(_ values: [String: PluginCellValue]) async throws { throw PluginImportError.importFailed("Row-based import is not supported by this connection") } diff --git a/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index a5e47d926e..e2b6dd1274 100644 --- a/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -6,6 +6,7 @@ import Foundation import os import TableProPluginKit +import TableProSQLGrammar final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable { let databaseTypeId: String @@ -20,6 +21,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable /// construction, on the main actor, because the registry lives there and this is asked for from /// the export plugin's own thread. let supportsCascadeDrop: Bool + let lexicalFeatures: SQLLexicalFeatures private let implicitSchemaName: String? private let pagination: PaginationCapability private let cappedTables = OSAllocatedUnfairLock<[String]>(initialState: []) @@ -27,6 +29,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable init(driver: DatabaseDriver, databaseType: DatabaseType) { let snapshot = PluginMetadataRegistry.shared.snapshot(for: databaseType) self.supportsCascadeDrop = snapshot?.capabilities.supportsCascadeDrop ?? false + self.lexicalFeatures = databaseType.lexicalGrammar.pluginFeatures self.implicitSchemaName = snapshot?.schema.implicitSchemaName self.pagination = PaginationCapability.of(databaseType) self.driver = driver diff --git a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index 1780a01d4a..8ae4d14c59 100644 --- a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -6,6 +6,7 @@ import Foundation import os import TableProPluginKit +import TableProSQLGrammar final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { let databaseTypeId: String @@ -13,6 +14,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { private let driver: DatabaseDriver private let databaseType: DatabaseType + private let grammar: SQLLexicalGrammar private let columnMapping: [String: String] private let rowGenerator: SQLStatementGenerator? @@ -34,6 +36,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { self.isCancelled = isCancelled self.driver = driver self.databaseType = databaseType + self.grammar = databaseType.lexicalGrammar self.databaseTypeId = databaseType.rawValue self.targetTable = targetTable self.columnMapping = Dictionary( @@ -53,9 +56,32 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } func execute(statement: String) async throws { - _ = try await driver.execute(query: statement) + try await execute(statement: statement, line: 1) } + /// A SQL Server file arrives a batch at a time, as sqlcmd reads it, and goes to the server whole: T-SQL scopes a + /// variable, a table variable and a `TRY...CATCH` to one batch, and a routine's body runs to the end of its batch. + /// A driver that cannot send a batch whole runs its statements one by one, as the editor does. + func execute(statement: String, line: Int) async throws { + guard grammar.contains(.batchSeparatorLines) else { + _ = try await driver.execute(query: statement) + return + } + if driver.supportsResultSetBatches, + let answer = try await driver.executeBatch(query: statement, rowCap: Self.batchRowCap, parameters: nil) { + guard let failure = BatchErrorText.describe(answer.errors, batchStartLine: line) else { return } + throw DatabaseError.queryFailed(failure) + } + for batchStatement in SQLStatementScanner.executableStatements(in: statement, grammar: grammar) { + guard !isCancelled() else { throw PluginImportCancellationError() } + _ = try await driver.execute(query: batchStatement.sql) + } + } + + /// An import shows no rows, so a batch keeps as few of each result set as a driver takes, and the rest are read + /// past rather than held. + private static let batchRowCap = 1 + func insertRow(_ values: [String: PluginCellValue]) async throws { guard let targetTable else { throw PluginImportError.importFailed("No target table configured for row import") diff --git a/TablePro/Core/Utilities/SQL/SQLFileBatchLines.swift b/TablePro/Core/Utilities/SQL/SQLFileBatchLines.swift new file mode 100644 index 0000000000..5a441daf73 --- /dev/null +++ b/TablePro/Core/Utilities/SQL/SQLFileBatchLines.swift @@ -0,0 +1,62 @@ +// +// SQLFileBatchLines.swift +// TablePro +// + +import Foundation + +/// What the import parser needs to know about lines to cut a SQL Server script at its `GO` lines, read off a buffer +/// that holds only the part of the file the last chunks brought in. +enum SQLFileBatchLines { + private static let lineFeed: unichar = 0x0A + private static let carriageReturn: unichar = 0x0D + private static let space: unichar = 0x20 + private static let tab: unichar = 0x09 + + /// Whether only spaces and tabs follow the last line break once the units in `start.. Bool { + var index = end - 1 + while index >= start { + let unit = buffer.character(at: index) + if unit == lineFeed || unit == carriageReturn { + return true + } + if unit != space && unit != tab { + return false + } + index -= 1 + } + return before + } + + /// The first line break at or after `start`, or nil when the buffer ends first. + static func lineBreak(in buffer: NSString, from start: Int, length: Int) -> Int? { + var index = start + while index < length { + let unit = buffer.character(at: index) + if unit == lineFeed || unit == carriageReturn { + return index + } + index += 1 + } + return nil + } + + /// How many line feeds the blanks at the start of `text` hold, which is how far below its first line the text the + /// server receives begins once those blanks are trimmed. + static func leadingLineFeeds(in text: NSString?) -> Int { + guard let text else { return 0 } + var count = 0 + var index = 0 + while index < text.length { + let unit = text.character(at: index) + guard let scalar = Unicode.Scalar(unit), CharacterSet.whitespacesAndNewlines.contains(scalar) else { break } + if unit == lineFeed { + count += 1 + } + index += 1 + } + return count + } +} diff --git a/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 215f71208a..adff19bbeb 100644 --- a/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -11,6 +11,27 @@ import TableProSQLGrammar final class SQLFileParser: Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLFileParser") + /// SQL Server takes at most 65,536 packets of 4,096 bytes in one request, about 134 million UTF-16 units of batch + /// text: measured on Azure SQL Edge 15, a 133,900,000-unit batch ran and a 133,960,000-unit one closed the + /// connection. A batch that reaches half of that ends at its next `;`, which leaves the statement it ends inside + /// as much room again. A dump written with no `GO` line is then sent in pieces the server takes, and held one + /// piece at a time rather than whole. + static let defaultBatchCutLength = 67_108_864 + + private let batchCutLength: Int + + init(batchCutLength: Int = SQLFileParser.defaultBatchCutLength) { + self.batchCutLength = batchCutLength + } + + /// One statement the file holds, or on a grammar that cuts scripts into batches, one batch, with how many times + /// the script runs it. + private struct ParsedStatement { + let statement: String + let lineNumber: Int + let repeatCount: Int + } + private enum ParserState { case normal case inSingleLineComment @@ -45,6 +66,8 @@ final class SQLFileParser: Sendable { private static let kSmallM: unichar = 0x6D private static let kOpenBracket: unichar = 0x5B private static let kCloseBracket: unichar = 0x5D + private static let kCapitalG: unichar = 0x47 + private static let kSmallG: unichar = 0x67 nonisolated private static func needsLookahead( _ char: unichar, @@ -137,12 +160,32 @@ final class SQLFileParser: Sendable { var dollarTag: String = "" var quoteChar: unichar = 0 var backslashEscapesActive = false - var collected: [(statement: String, lineNumber: Int)] = [] + var collected: [ParsedStatement] = [] + + /// SQL Server's tools read a script the way sqlcmd does: a line holding only `GO` ends a batch, the batch goes + /// to the server whole with its `;` and its comments, and the `GO` line goes nowhere. The comments stay so the + /// server's line numbers count the file's lines and a routine keeps the comments written inside it. + let readsBatches: Bool + let batchCutLength: Int + + /// Whether only spaces and tabs stand between the last line break and the unit being read, which is where a + /// `GO` line can start. Tracked here because the buffer no longer holds the line's start once a chunk is + /// consumed. + var lineHoldsOnlyBlanks = true + + /// How much of the line after a `G` held back for the rest of its line is already known to hold no line break, + /// so a long line is searched once rather than once per chunk. + var separatorLineSearched = 0 + + /// The line the batch text starts on before its leading whitespace, and how many units were read before it. + var batchTextStartLine = 1 + var batchStartUnit = 0 + var unitsBeforeBuffer = 0 /// The statement grammar, for a dialect whose statement can own its `;`: a PL/SQL unit arrives whole with - /// its own `;`, and a T-SQL `MERGE` with the `;` SQL Server refuses to run it without, so a routine body - /// arrives whole there too. Every other dialect has always split an import at each `;` and relies on - /// `DELIMITER` or dollar quoting for a routine body, and keeps doing so. + /// its own `;`. A batch keeps every `;` it holds, a T-SQL `MERGE`'s included, so a grammar read in batches + /// needs none. Every other dialect has always split an import at each `;` and relies on `DELIMITER` or dollar + /// quoting for a routine body, and keeps doing so. var boundaries: (any SQLStatementBoundaryTracking)? var word: [unichar] = [] var alternativeQuoteCloser: unichar = 0 @@ -162,13 +205,21 @@ final class SQLFileParser: Sendable { /// the one after it is settled with nothing after it, instead of being left in the buffer and dropped. var atEndOfInput = false - init(grammar: SQLLexicalGrammar, currentStatement: NSMutableString?) { + init(grammar: SQLLexicalGrammar, currentStatement: NSMutableString?, batchCutLength: Int) { self.grammar = grammar self.currentStatement = currentStatement - self.boundaries = SQLStatementBoundaries.statementsCanOwnTerminator(in: grammar) + self.readsBatches = grammar.contains(.batchSeparatorLines) + self.batchCutLength = batchCutLength + self.boundaries = !readsBatches && SQLStatementBoundaries.statementsCanOwnTerminator(in: grammar) ? SQLStatementBoundaries.makeTracker(for: grammar) : nil } + + /// Whether the block comment being read goes into the statement: a conditional comment is SQL the server runs, + /// and a batch keeps every comment. + var keepsCommentText: Bool { + readsBatches || isConditionalComment + } } private static func trimmedStatement(_ ctx: ParserContext) -> String { @@ -326,6 +377,11 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: false) } + if ctx.readsBatches, ctx.lineHoldsOnlyBlanks, char == kCapitalG || char == kSmallG, + let step = endBatchAtSeparatorLine(&ctx, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) { + return step + } + switch observeWordCharacter(&ctx, char: char, i: i, nsBuffer: nsBuffer, bufLen: bufLen) { case .continues: (ctx.hasStatementContent, ctx.statementStartLine) = markContent( @@ -361,6 +417,9 @@ final class SQLFileParser: Sendable { ) { ctx.state = .inSingleLineComment ctx.boundaries?.observeGap() + if ctx.readsBatches { + appendRange(&ctx, from: i, to: i + 2, in: nsBuffer) + } i += 2 return StepResult(advanced: true, deferred: false) } @@ -370,6 +429,9 @@ final class SQLFileParser: Sendable { || (char == kSlash && nextChar == kSlash && ctx.grammar.contains(.doubleSlashLineComments)) { ctx.state = .inSingleLineComment ctx.boundaries?.observeGap() + if ctx.readsBatches { + appendChar(char, to: ctx.currentStatement) + } return StepResult(advanced: false, deferred: false) } @@ -381,6 +443,8 @@ final class SQLFileParser: Sendable { if ctx.isConditionalComment { (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) + } + if ctx.keepsCommentText { appendChar(char, to: ctx.currentStatement) appendChar(next, to: ctx.currentStatement) } else { @@ -452,7 +516,7 @@ final class SQLFileParser: Sendable { } if ctx.isSingleCharDelimiter && char == kSemicolon { - processSemicolon(&ctx) + processSemicolon(&ctx, at: i) return StepResult(advanced: false, deferred: false) } @@ -482,8 +546,17 @@ final class SQLFileParser: Sendable { } /// A `;` ends the statement unless the grammar holds it inside a PL/SQL unit or a routine body, and a statement - /// that owns the `;` that ends it keeps it. - private static func processSemicolon(_ ctx: inout ParserContext) { + /// that owns the `;` that ends it keeps it. In a batch it ends nothing, unless the batch has grown past what the + /// server takes in one request. + private static func processSemicolon(_ ctx: inout ParserContext, at i: Int) { + if ctx.readsBatches { + appendChar(kSemicolon, to: ctx.currentStatement) + let batchEnd = ctx.unitsBeforeBuffer + i + 1 + if batchEnd - ctx.batchStartUnit >= ctx.batchCutLength { + yieldBatch(&ctx, repeatCount: 1, nextBatchStart: batchEnd) + } + return + } guard ctx.boundaries != nil else { yieldAndReset(&ctx) return @@ -668,27 +741,93 @@ final class SQLFileParser: Sendable { private static func yieldAndReset(_ ctx: inout ParserContext) { if ctx.hasStatementContent { let text = trimmedStatement(ctx) - ctx.collected.append((text, ctx.statementStartLine)) + ctx.collected.append(ParsedStatement(statement: text, lineNumber: ctx.statementStartLine, repeatCount: 1)) } resetStatement(&ctx) } + /// Ends the batch read so far, which is sent only when it holds code: a batch of comments and blanks runs nothing. + /// + /// Its line is the one its text starts on once the leading blanks are trimmed, because that is the line the server + /// counts as the batch's first. + private static func yieldBatch(_ ctx: inout ParserContext, repeatCount: Int, nextBatchStart: Int) { + if ctx.hasStatementContent { + let lineNumber = ctx.batchTextStartLine + SQLFileBatchLines.leadingLineFeeds(in: ctx.currentStatement) + ctx.collected.append( + ParsedStatement(statement: trimmedStatement(ctx), lineNumber: lineNumber, repeatCount: repeatCount) + ) + } + resetStatement(&ctx) + ctx.batchTextStartLine = ctx.currentLine + ctx.batchStartUnit = nextBatchStart + } + + /// Ends the batch at the `GO` line starting at `i` and steps over the line, or answers nil when the line is code. + private static func endBatchAtSeparatorLine( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult? { + switch batchSeparator(&ctx, at: i, nsBuffer: nsBuffer, bufLen: bufLen) { + case .needsMoreData: + return StepResult(advanced: false, deferred: true) + case .separator(let separator): + let end = NSMaxRange(separator.range) + yieldBatch(&ctx, repeatCount: separator.repeatCount, nextBatchStart: ctx.unitsBeforeBuffer + end) + i = end + return StepResult(advanced: true, deferred: false) + case .code: + return nil + } + } + + private enum BatchSeparatorRead { + case separator(SQLBatchSeparator) + case code + case needsMoreData + } + + /// Reads the line a `G` starts once the buffer holds all of it: what follows `GO` decides, and a line cut off at + /// the end of a chunk would read as ending there. + private static func batchSeparator( + _ ctx: inout ParserContext, + at i: Int, + nsBuffer: NSString, + bufLen: Int + ) -> BatchSeparatorRead { + let lineBreak = SQLFileBatchLines.lineBreak(in: nsBuffer, from: i + ctx.separatorLineSearched, length: bufLen) + guard lineBreak != nil || ctx.atEndOfInput else { + ctx.separatorLineSearched = bufLen - i + return .needsMoreData + } + ctx.separatorLineSearched = 0 + guard let separator = SQLBatchSeparator.line(startingAt: i, in: nsBuffer, length: bufLen, grammar: ctx.grammar) + else { + return .code + } + return .separator(separator) + } + private static func processMultiLineComment( _ ctx: inout ParserContext, char: unichar, nextChar: unichar?, i: inout Int ) -> Bool { - if ctx.isConditionalComment { + if ctx.keepsCommentText { appendChar(char, to: ctx.currentStatement) } if char == kSlash, nextChar == kStar, !ctx.isConditionalComment, ctx.grammar.contains(.nestedBlockComments) { + if ctx.readsBatches { + appendChar(kStar, to: ctx.currentStatement) + } ctx.commentDepth += 1 i += 2 return true } if char == kStar, let next = nextChar, next == kSlash { - if ctx.isConditionalComment { + if ctx.keepsCommentText { appendChar(next, to: ctx.currentStatement) } ctx.commentDepth -= 1 @@ -815,15 +954,18 @@ final class SQLFileParser: Sendable { return StepResult(advanced: true, deferred: false) } + /// The file's statements in order, each as many times as the script runs it: a batch ended by `GO 5` comes five + /// times, one run each, handed out one at a time rather than copied. func parseFile( url: URL, encoding: String.Encoding, - grammar: SQLLexicalGrammar, - countOnly: Bool = false + grammar: SQLLexicalGrammar ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { - let session = ParseSession(url: url, encoding: encoding, grammar: grammar, countOnly: countOnly) + let session = ParseSession( + url: url, encoding: encoding, grammar: grammar, countOnly: false, batchCutLength: batchCutLength + ) return AsyncThrowingStream(unfolding: { - try await session.next() + try await session.nextRun() }) } @@ -839,15 +981,18 @@ final class SQLFileParser: Sendable { private var decoder: SQLChunkDecoder private var emitIndex = 0 private var finished = false + private var repeating: ParsedStatement? + private var remainingRuns = 0 - init(url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar, countOnly: Bool) { + init(url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar, countOnly: Bool, batchCutLength: Int) { self.url = url self.encoding = encoding self.grammar = grammar self.decoder = SQLChunkDecoder(encoding: encoding) self.ctx = ParserContext( grammar: grammar, - currentStatement: countOnly ? nil : NSMutableString() + currentStatement: countOnly ? nil : NSMutableString(), + batchCutLength: batchCutLength ) } @@ -855,7 +1000,18 @@ final class SQLFileParser: Sendable { closeFile() } - func next() async throws -> (statement: String, lineNumber: Int)? { + func nextRun() async throws -> (statement: String, lineNumber: Int)? { + if let repeating, remainingRuns > 0 { + remainingRuns -= 1 + return (repeating.statement, repeating.lineNumber) + } + guard let next = try await nextStatement() else { return nil } + repeating = next + remainingRuns = next.repeatCount - 1 + return (next.statement, next.lineNumber) + } + + func nextStatement() async throws -> ParsedStatement? { while true { if emitIndex < ctx.collected.count { let item = ctx.collected[emitIndex] @@ -920,6 +1076,7 @@ final class SQLFileParser: Sendable { var i = 0 while i < bufLen { + let stepStart = i let char = nsBuffer.character(at: i) let nextChar: unichar? = (i + 1 < bufLen) ? nsBuffer.character(at: i + 1) : nil @@ -946,6 +1103,9 @@ final class SQLFileParser: Sendable { shouldDefer = result.deferred case .inSingleLineComment: + if ctx.readsBatches { + SQLFileParser.appendChar(char, to: ctx.currentStatement) + } if char == SQLFileParser.kNewline || (char == SQLFileParser.kCarriageReturn && grammar.contains(.carriageReturnEndsLineComments)) { @@ -1017,8 +1177,14 @@ final class SQLFileParser: Sendable { if ctx.state != .normal { ctx.previousUnitInWord = false } + if ctx.readsBatches { + ctx.lineHoldsOnlyBlanks = SQLFileBatchLines.holdsOnlyBlanks( + nsBuffer, from: stepStart, to: min(i, bufLen), before: ctx.lineHoldsOnlyBlanks + ) + } } + ctx.unitsBeforeBuffer += min(i, bufLen) if i < bufLen { nsBuffer.deleteCharacters(in: NSRange(location: 0, length: i)) } else { @@ -1029,10 +1195,16 @@ final class SQLFileParser: Sendable { private func emitTrailingStatement() { ctx.pendingSlashLine = false ctx.pendingSlashTrailing.removeAll() + if ctx.readsBatches { + SQLFileParser.yieldBatch(&ctx, repeatCount: 1, nextBatchStart: ctx.unitsBeforeBuffer) + return + } guard ctx.hasStatementContent else { return } let text = SQLFileParser.trimmedStatement(ctx) if SQLFileParser.extractDelimiterChange(text) == nil { - ctx.collected.append((text, ctx.statementStartLine)) + ctx.collected.append( + ParsedStatement(statement: text, lineNumber: ctx.statementStartLine, repeatCount: 1) + ) } } @@ -1057,16 +1229,21 @@ final class SQLFileParser: Sendable { } } + /// How many statements the import runs, a batch ended by `GO 5` counting five times. The runs are added up rather + /// than walked, because a count can reach `Int32.max`. func countStatements( url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar ) async throws -> Int { + let session = ParseSession( + url: url, encoding: encoding, grammar: grammar, countOnly: true, batchCutLength: batchCutLength + ) var count = 0 - for try await _ in parseFile(url: url, encoding: encoding, grammar: grammar, countOnly: true) { + while let statement = try await session.nextStatement() { try Task.checkCancellation() - count += 1 + count += statement.repeatCount } return count diff --git a/TableProTests/Core/Plugins/SQLServerImportBatchTests.swift b/TableProTests/Core/Plugins/SQLServerImportBatchTests.swift new file mode 100644 index 0000000000..0705678f93 --- /dev/null +++ b/TableProTests/Core/Plugins/SQLServerImportBatchTests.swift @@ -0,0 +1,189 @@ +// +// SQLServerImportBatchTests.swift +// TableProTests +// +// A SQL file imported into SQL Server reaches the server a batch at a time, the way sqlcmd sends it. The import used +// to split it at every `;` and send each `GO` line on to the server, which ran the statement after a `GO`, refused the +// `GO` itself with Msg 2812 or Msg 102, and lost every variable between the statement that declared it and the next. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class BatchImportDriver: PluginDatabaseDriver, @unchecked Sendable { + let declaresBatches: Bool + private(set) var sentBatches: [(query: String, rowCap: Int?)] = [] + private(set) var executedStatements: [String] = [] + + init(declaresBatches: Bool) { + self.declaresBatches = declaresBatches + } + + var capabilities: PluginCapabilities { + declaresBatches ? [.resultSetBatches] : [] + } + + func connect() async throws {} + func disconnect() {} + + func execute(query: String) async throws -> PluginQueryResult { + executedStatements.append(query) + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + /// Answers the way SQL Server does for a batch naming a table that does not exist: the error comes back with the + /// batch's own line, counted from the batch's first line, and the rest of the batch still ran. + func executeBatch(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginBatchResult? { + guard declaresBatches else { return nil } + sentBatches.append((query, rowCap)) + let lines = query.components(separatedBy: "\n") + let errors = lines.enumerated() + .filter { $0.element.contains("missing") } + .map { index, _ in + PluginBatchError( + message: "Invalid object name 'missing'.", + code: 208, + line: index + 1, + procedure: nil, + precedingResultSetCount: 0 + ) + } + return PluginBatchResult( + resultSets: [], + rowsAffected: 1, + errors: errors, + discardedResultSetCount: 0, + executionTime: 0 + ) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +/// Serialized because `SQLImportPlugin.settings` persists through plugin storage. +@Suite("SQL Server import runs each GO batch whole", .serialized) +struct SQLServerImportBatchTests { + private func makeSink( + declaresBatches: Bool, + databaseType: DatabaseType = .mssql + ) -> (ImportDataSinkAdapter, BatchImportDriver) { + let driver = BatchImportDriver(declaresBatches: declaresBatches) + let adapter = PluginDriverAdapter( + connection: DatabaseConnection(name: "Test", type: databaseType), + pluginDriver: driver + ) + return (ImportDataSinkAdapter(driver: adapter, databaseType: databaseType), driver) + } + + private func runImport(_ script: String, sink: ImportDataSinkAdapter) async throws -> PluginImportResult { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try script.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + let plugin = SQLImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + plugin.settings.errorHandling = .stopAndRollback + plugin.settings.wrapInTransaction = false + plugin.settings.disableForeignKeyChecks = false + + return try await plugin.performImport( + source: SqlFileImportSource(url: url, encoding: .utf8, grammar: DatabaseType.mssql.lexicalGrammar), + sink: sink, + progress: PluginImportProgress(progress: Progress()) + ) + } + + @Test("A batch goes to the driver whole and once, keeping no rows") + func batchIsSentWhole() async throws { + let (sink, driver) = makeSink(declaresBatches: true) + let batch = "DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);" + try await sink.execute(statement: batch, line: 4) + #expect(driver.sentBatches.map(\.query) == [batch]) + #expect(driver.sentBatches.map(\.rowCap) == [1]) + #expect(driver.executedStatements.isEmpty) + } + + @Test("An error inside a batch fails it on the file's own line") + func batchErrorNamesTheFileLine() async throws { + let (sink, _) = makeSink(declaresBatches: true) + do { + try await sink.execute(statement: "SELECT 1\nSELECT * FROM missing", line: 10) + Issue.record("A batch that raised an error was reported as run") + } catch { + #expect(error.localizedDescription == "Line 11: Invalid object name 'missing'.") + } + } + + @Test("A driver that cannot send a batch whole runs its statements one by one") + func batchlessDriverRunsStatements() async throws { + let (sink, driver) = makeSink(declaresBatches: false) + try await sink.execute(statement: "DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);", line: 1) + #expect(driver.executedStatements == ["DECLARE @x INT = 1", "INSERT INTO t (v) VALUES (@x)"]) + } + + @Test("Another engine's statement is sent as it came") + func otherEnginesSendTheStatement() async throws { + let (sink, driver) = makeSink(declaresBatches: true, databaseType: .mysql) + try await sink.execute(statement: "INSERT INTO t VALUES (1)", line: 3) + #expect(driver.executedStatements == ["INSERT INTO t VALUES (1)"]) + #expect(driver.sentBatches.isEmpty) + } + + @Test("An imported Compare script sends each batch whole and never a GO line") + func compareScriptImports() async throws { + let (sink, driver) = makeSink(declaresBatches: true) + let script = """ + DROP PROCEDURE [dbo].[p]; + GO + CREATE PROCEDURE dbo.p AS SET NOCOUNT ON; SELECT 1; + GO + INSERT INTO t VALUES (1) + GO 2 + """ + let result = try await runImport(script, sink: sink) + #expect(driver.sentBatches.map(\.query) == [ + "DROP PROCEDURE [dbo].[p];", + "CREATE PROCEDURE dbo.p AS SET NOCOUNT ON; SELECT 1;", + "INSERT INTO t VALUES (1)", + "INSERT INTO t VALUES (1)", + ]) + #expect(driver.executedStatements.isEmpty) + #expect(result.executedStatements == 4) + } + + @Test("A failed batch stops the import on the batch's line, with the error's line in the file") + func failedBatchReportsBothLines() async throws { + let (sink, _) = makeSink(declaresBatches: true) + let script = "SELECT 1\nGO\n-- lead\nSELECT 2\nSELECT * FROM missing\nGO\nSELECT 3" + do { + _ = try await runImport(script, sink: sink) + Issue.record("An import whose batch raised an error completed") + } catch let PluginImportError.statementFailed(statement, line, underlying) { + #expect(statement == "-- lead\nSELECT 2\nSELECT * FROM missing") + #expect(line == 3) + #expect(underlying.localizedDescription == "Line 5: Invalid object name 'missing'.") + } + } + + @Test("A script with no GO line keeps a declared variable for the statement that reads it") + func declaredVariableReachesItsReader() async throws { + let (sink, driver) = makeSink(declaresBatches: true) + _ = try await runImport("DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);\n", sink: sink) + #expect(driver.sentBatches.map(\.query) == ["DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);"]) + } +} diff --git a/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift b/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift new file mode 100644 index 0000000000..aba6243b5b --- /dev/null +++ b/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift @@ -0,0 +1,236 @@ +// +// SQLFileParserBatchTests.swift +// TableProTests +// +// A SQL Server script imported from a file is read the way sqlcmd reads it: a line holding only `GO` ends a batch, +// the batch reaches the server whole, and the `GO` line reaches it not at all. The parser streams the file in 64 KiB +// chunks, so a `GO` line can be cut in two by a chunk boundary, and the split must not depend on where it falls. +// + +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +@Suite("SQLFileParser - SQL Server batches") +struct SQLFileParserBatchTests { + private struct Run: Equatable { + let statement: String + let line: Int + } + + private static let chunkSize = 65_536 + + private static func write(_ sql: String) throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + return url + } + + private static func runs( + _ sql: String, + grammar: SQLLexicalGrammar = TestGrammar.sqlServer, + parser: SQLFileParser = SQLFileParser() + ) async throws -> [Run] { + let url = try write(sql) + defer { try? FileManager.default.removeItem(at: url) } + var runs: [Run] = [] + for try await (statement, line) in parser.parseFile(url: url, encoding: .utf8, grammar: grammar) { + runs.append(Run(statement: statement, line: line)) + } + return runs + } + + private static func statements(_ sql: String, grammar: SQLLexicalGrammar = TestGrammar.sqlServer) async throws + -> [String] { + try await runs(sql, grammar: grammar).map(\.statement) + } + + private static func count(_ sql: String, parser: SQLFileParser = SQLFileParser()) async throws -> Int { + let url = try write(sql) + defer { try? FileManager.default.removeItem(at: url) } + return try await parser.countStatements(url: url, encoding: .utf8, grammar: TestGrammar.sqlServer) + } + + @Test("SQL Server is the engine whose scripts are cut at GO lines") + func sqlServerReadsBatches() { + #expect(TestGrammar.sqlServer.contains(.batchSeparatorLines)) + } + + @Test("A Compare script's procedure arrives whole, with its inner semicolon, and no GO reaches the driver") + func compareScript() async throws { + let script = """ + DROP PROCEDURE [dbo].[p]; + GO + CREATE PROCEDURE dbo.p AS SET NOCOUNT ON; SELECT 1; + GO + + """ + #expect(try await Self.runs(script) == [ + Run(statement: "DROP PROCEDURE [dbo].[p];", line: 1), + Run(statement: "CREATE PROCEDURE dbo.p AS SET NOCOUNT ON; SELECT 1;", line: 3), + ]) + } + + @Test("An SSMS script with no semicolons is cut at its GO lines alone") + func ssmsScript() async throws { + let script = """ + SET ANSI_NULLS ON + GO + CREATE TABLE [dbo].[t]([id] [int] NOT NULL) + GO + INSERT [dbo].[t] ([id]) VALUES (1) + INSERT [dbo].[t] ([id]) VALUES (2) + GO + """ + #expect(try await Self.runs(script) == [ + Run(statement: "SET ANSI_NULLS ON", line: 1), + Run(statement: "CREATE TABLE [dbo].[t]([id] [int] NOT NULL)", line: 3), + Run(statement: "INSERT [dbo].[t] ([id]) VALUES (1)\nINSERT [dbo].[t] ([id]) VALUES (2)", line: 5), + ]) + } + + @Test("A script with no GO line is one batch, so a variable is still declared where it is read") + func scriptWithoutGoIsOneBatch() async throws { + let script = "DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);\n" + #expect(try await Self.statements(script) == ["DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);"]) + } + + @Test("GO n runs the batch n times, and the count says so") + func repeatCount() async throws { + let script = "INSERT t VALUES (1)\nGO 3\nSELECT 2\nGO\n" + #expect(try await Self.statements(script) == [ + "INSERT t VALUES (1)", "INSERT t VALUES (1)", "INSERT t VALUES (1)", "SELECT 2", + ]) + #expect(try await Self.count(script) == 4) + } + + @Test("A count as large as GO allows is added up, not walked") + func largestRepeatCountIsCounted() async throws { + #expect(try await Self.count("INSERT t VALUES (1)\nGO 2147483647\nSELECT 1") == 2_147_483_648) + } + + @Test("GO is read case-insensitively, after blanks, with a count and a trailing comment") + func acceptedSpellings() async throws { + let script = "SELECT 1\n go\nSELECT 2\n\tGo 2 -- twice\nSELECT 3\nGO--glued\nSELECT 4" + #expect(try await Self.statements(script) == ["SELECT 1", "SELECT 2", "SELECT 2", "SELECT 3", "SELECT 4"]) + } + + @Test( + "A line that holds anything else is not a separator and stays in the batch", + arguments: ["GO;", "GO 0", "GOTO done", "go_table", "GO5", "GO /* c */", "SELECT 1 GO", "/* c */ GO"] + ) + func rejectedLines(line: String) async throws { + let script = "SELECT 1\n\(line)\nSELECT 2" + #expect(try await Self.statements(script) == [script]) + } + + @Test("GO inside a literal, a quoted identifier or a comment separates nothing, across lines too", arguments: [ + "SELECT 'a\nGO\nb'", + "SELECT N'a\nGO\nb'", + "SELECT [a\nGO\nb] FROM t", + "SELECT \"a\nGO\nb\" FROM t", + "/* a\nGO\n*/ SELECT 1", + "/* outer /* inner */\nGO\nstill outer */ SELECT 1", + "SELECT 1 -- note\nGO_ON\nSELECT 2", + ]) + func goInsideNonCode(script: String) async throws { + #expect(try await Self.statements(script) == [script]) + } + + @Test("An unterminated block comment swallows every GO line after it, as sqlcmd reads it") + func unterminatedCommentSwallowsGo() async throws { + let script = "SELECT 1\n/* open\nGO\nSELECT 2\nGO" + #expect(try await Self.statements(script) == [script]) + } + + @Test("Comments stay in the batch, so the server's line numbers count the file's lines") + func commentsAndLinesAreKept() async throws { + let script = """ + -- header + + SELECT 1 + GO + + /* note + spans */ + SELECT 2 -- tail + GO 2 + """ + #expect(try await Self.runs(script) == [ + Run(statement: "-- header\n\nSELECT 1", line: 1), + Run(statement: "/* note\n spans */\nSELECT 2 -- tail", line: 6), + Run(statement: "/* note\n spans */\nSELECT 2 -- tail", line: 6), + ]) + } + + @Test("A batch of nothing but comments and blanks runs nothing, and neither do GO lines in a row") + func emptyBatchesAreSkipped() async throws { + let script = "GO\n-- only a comment\nGO\n\nGO 5\nSELECT 1\nGO\n/* trailing */" + #expect(try await Self.runs(script) == [Run(statement: "SELECT 1", line: 6)]) + #expect(try await Self.count(script) == 1) + } + + @Test("Carriage returns end a GO line as line feeds do") + func carriageReturns() async throws { + #expect(try await Self.statements("SELECT 1\r\nGO\r\nSELECT 2\r\nGO 2\r\n") == ["SELECT 1", "SELECT 2", "SELECT 2"]) + #expect(try await Self.statements("SELECT 1\rGO\rSELECT 2") == ["SELECT 1", "SELECT 2"]) + } + + @Test("A GO line may end the file, with no line break after it") + func goAtTheEnd() async throws { + #expect(try await Self.statements("SELECT 1\nGO") == ["SELECT 1"]) + #expect(try await Self.statements("SELECT 1\nGO 2") == ["SELECT 1", "SELECT 1"]) + } + + @Test("A chunk boundary anywhere around a GO line changes nothing") + func chunkBoundaryAnywhere() async throws { + let script = "SELECT 'a\nb' AS x;\nGO 2 -- two\nSELECT 3 GO\n GOTO done\nGO\nSELECT 4" + for boundary in 0..<(script as NSString).length { + let padding = "--" + String(repeating: "x", count: Self.chunkSize - boundary - 3) + "\n" + let statements = try await Self.statements(padding + script) + let first = padding + "SELECT 'a\nb' AS x;" + #expect(statements == [first, first, "SELECT 3 GO\n GOTO done", "SELECT 4"], "boundary \(boundary)") + } + } + + @Test("A GO after a literal that closes on its line is code, wherever the chunk boundary falls") + func goAfterALiteralOnItsLine() async throws { + let script = "SELECT 'a\n'GO\nSELECT 2" + for boundary in 0..<(script as NSString).length { + let padding = "--" + String(repeating: "x", count: Self.chunkSize - boundary - 3) + "\n" + #expect(try await Self.statements(padding + script) == [padding + script], "boundary \(boundary)") + } + } + + @Test("A batch past the cut length ends at its next semicolon outside a literal, and the count agrees") + func longBatchIsCutAtASemicolon() async throws { + let parser = SQLFileParser(batchCutLength: 20) + let script = "INSERT t VALUES ('a;b');\nINSERT t VALUES (2);\nINSERT t VALUES (3)\nGO\nSELECT 4;" + #expect(try await Self.statements(script, parser: parser) == [ + "INSERT t VALUES ('a;b');", + "INSERT t VALUES (2);", + "INSERT t VALUES (3)", + "SELECT 4;", + ]) + #expect(try await Self.count(script, parser: parser) == 4) + } + + @Test("A batch below the cut length keeps every statement together") + func shortBatchIsNotCut() async throws { + let script = "INSERT t VALUES (1);\nINSERT t VALUES (2);\nINSERT t VALUES (3);" + #expect(try await Self.statements(script) == [script]) + } + + @Test("An engine without batches still splits at each semicolon and drops its comments") + func otherEnginesAreUnchanged() async throws { + let script = "-- note\nSELECT 1;\nGO\nSELECT 2;" + #expect(try await Self.statements(script, grammar: TestGrammar.postgres) == ["SELECT 1", "GO\nSELECT 2"]) + } +} + +private extension SQLFileParserBatchTests { + static func statements(_ sql: String, parser: SQLFileParser) async throws -> [String] { + try await runs(sql, parser: parser).map(\.statement) + } +} diff --git a/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index 310b03c64a..d0ea3114f5 100644 --- a/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -202,7 +202,8 @@ struct SQLFileParserTests { MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, v) VALUES (s.id, s.v) """ - /// SQL import sends each statement on its own, and SQL Server fails a `MERGE` sent without its `;` with Msg 10713. + /// SQL import sends a SQL Server file a batch at a time with its text as written, so a `MERGE` keeps the `;` SQL + /// Server fails it without, with Msg 10713. @Test("SQL Server: a MERGE keeps its ; in the middle of a file and at its end") func sqlServerMergeKeepsItsTerminator() async throws { let sql = """ @@ -212,44 +213,30 @@ struct SQLFileParserTests { TRUNCATE TABLE dbo.\u{6CE8}\u{6587} \(Self.merge); """ - let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) - #expect(stmts == [ - "UPDATE dbo.s SET v = 1", - Self.merge + ";", - "SELECT COUNT(*) FROM dbo.t", - "TRUNCATE TABLE dbo.\u{6CE8}\u{6587}\n" + Self.merge + ";", - ]) + #expect(try await Self.parse(sql, grammar: TestGrammar.sqlServer) == [sql]) } - /// SQL Server reads `1MERGE` as a number and a `MERGE`, as the statement scanner does. - @Test("SQL Server: a MERGE glued to a number keeps its ;") - func sqlServerMergeGluedToNumberKeepsItsTerminator() async throws { - let sql = "SELECT 1\(Self.merge);\nSELECT 1.\(Self.merge);" - let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) - #expect(stmts == ["SELECT 1\(Self.merge);", "SELECT 1.\(Self.merge);"]) - } - - @Test("SQL Server: a name that ends in merge is not a MERGE") - func sqlServerNameEndingInMergeDropsTheSeparator() async throws { - let sql = "SELECT 1 AS x$merge;\nSELECT 1 AS \u{00E9}merge;\nSELECT a FROM #merge;" - let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) - #expect(stmts == ["SELECT 1 AS x$merge", "SELECT 1 AS \u{00E9}merge", "SELECT a FROM #merge"]) + @Test("SQL Server: a file with no GO line arrives as written, every ; kept", arguments: [ + "SELECT 1\(merge);\nSELECT 1.\(merge);", + "SELECT 1 AS x$merge;\nSELECT 1 AS \u{00E9}merge;\nSELECT a FROM #merge;", + ]) + func sqlServerScriptArrivesAsWritten(sql: String) async throws { + #expect(try await Self.parse(sql, grammar: TestGrammar.sqlServer) == [sql]) } @Test("SQL Server: a MERGE whose keyword straddles two read chunks keeps its ;") func sqlServerMergeAcrossChunkBoundary() async throws { let padding = String(repeating: "a", count: 65_523) let sql = "SELECT '\(padding)';\n\(Self.merge);" - let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) - #expect(stmts == ["SELECT '\(padding)'", Self.merge + ";"]) + #expect(try await Self.parse(sql, grammar: TestGrammar.sqlServer) == [sql]) } - /// A T-SQL routine has no `DELIMITER` and no dollar quoting, so only the statement grammar keeps its body whole. + /// A T-SQL routine runs to the end of its batch, as sqlcmd sends it, so its body arrives whole. @Test("SQL Server: a procedure body arrives whole") func sqlServerProcedureBodyArrivesWhole() async throws { - let procedure = "CREATE PROCEDURE dbo.p AS BEGIN UPDATE dbo.t SET v = 1; \(Self.merge); END" - let stmts = try await Self.parse(procedure + ";\nSELECT 1;", grammar: TestGrammar.sqlServer) - #expect(stmts == [procedure, "SELECT 1"]) + let procedure = "CREATE PROCEDURE dbo.p AS BEGIN UPDATE dbo.t SET v = 1; \(Self.merge); END;" + let stmts = try await Self.parse(procedure + "\nGO\nSELECT 1;", grammar: TestGrammar.sqlServer) + #expect(stmts == [procedure, "SELECT 1;"]) } @Test("Postgres: the ; after a MERGE is still a separator") diff --git a/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift b/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift index 6b36523557..ee4c890e50 100644 --- a/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift @@ -257,6 +257,24 @@ struct SQLScriptTextTests { #expect(try await Self.imported(script, grammar: TestGrammar.mysql) == statements) } + /// The import reads a SQL Server script a batch at a time, the way sqlcmd reads it, so each statement a saved + /// script ends with a `GO` line comes back as a batch of its own with nothing cut out of it. + @Test("A SQL Server script imports one batch per statement it was written from") + func sqlServerScriptImports() async throws { + let statements = [ + "DROP PROCEDURE p", + "CREATE PROCEDURE p AS SET NOCOUNT ON; SELECT 1;", + "INSERT INTO t (a) VALUES ('a;b')", + ] + let script = Self.sqlServer.script(statements) + + #expect(try await Self.imported(script, grammar: TestGrammar.sqlServer) == [ + "DROP PROCEDURE p;", + "CREATE PROCEDURE p AS SET NOCOUNT ON; SELECT 1;", + "INSERT INTO t (a) VALUES ('a;b');", + ]) + } + private static func imported(_ sql: String, grammar: SQLLexicalGrammar) async throws -> [String] { let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") try sql.write(to: url, atomically: true, encoding: .utf8) diff --git a/TableProTests/Plugins/SQLExportBatchSeparatorTests.swift b/TableProTests/Plugins/SQLExportBatchSeparatorTests.swift new file mode 100644 index 0000000000..bb5c1ed84f --- /dev/null +++ b/TableProTests/Plugins/SQLExportBatchSeparatorTests.swift @@ -0,0 +1,153 @@ +// +// SQLExportBatchSeparatorTests.swift +// TableProTests +// +// SQL Server's client runs a script in batches cut at lines holding only `GO`, and SQL Server refuses a view, a +// routine or a trigger that is not the first statement of its batch with Msg 111. A dump written with only `;` +// between statements is one batch, so its first view failed the whole batch in sqlcmd, in SQL Server Management +// Studio, and in TablePro's own import once that import read SQL Server scripts in batches. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@Suite("SQL export for an engine that runs scripts in batches") +struct SQLExportBatchSeparatorTests { + private final class ServerDataSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId: String + let lexicalFeatures: SQLLexicalFeatures + private let scriptTextOwner: SQLScriptText + + init(databaseType: DatabaseType) { + self.databaseTypeId = databaseType.rawValue + self.lexicalFeatures = databaseType.lexicalGrammar.pluginFeatures + self.scriptTextOwner = SQLScriptText(databaseType: databaseType) + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.yield(.header(PluginStreamHeader(columns: ["id", "name"], columnTypeNames: ["INT", "NVARCHAR"]))) + continuation.yield(.rows([[.text("1"), .text("a")], [.text("2"), .text("b")]])) + continuation.finish() + } + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + [ + "orders": [ + PluginColumnInfo(name: "id", dataType: "INT", isNullable: false, identityKind: .byDefault), + PluginColumnInfo(name: "name", dataType: "NVARCHAR"), + ], + ] + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + "CREATE TABLE [orders] ([id] INT IDENTITY(1,1) NOT NULL, [name] NVARCHAR(10) NULL)" + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + switch object.kind { + case .view: + return "CREATE VIEW [v_orders] AS SELECT id FROM orders" + case .routine: + return "CREATE PROCEDURE [p_orders] AS SET NOCOUNT ON; SELECT 1;" + default: + return try await fetchTableDDL(table: object.name, databaseName: object.databaseName) + } + } + + func scriptText(for ddl: String) -> String { + scriptTextOwner.scriptText(forDriverText: ddl) + } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { + "[\(identifier.replacingOccurrences(of: "]", with: "]]"))]" + } + + func escapeStringLiteral(_ value: String) -> String { + value.replacingOccurrences(of: "'", with: "''") + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } + } + + private func object(_ name: String, kind: PluginExportObjectKind) -> PluginExportTable { + PluginExportTable( + name: name, + databaseName: "", + tableType: kind.rawValue, + optionValues: [true, true, true], + schema: nil, + kind: kind) + } + + private func dump(_ databaseType: DatabaseType) async throws -> String { + try await SQLExportHarness.shared.dump( + tables: [ + object("orders", kind: .table), + object("v_orders", kind: .view), + object("p_orders", kind: .routine), + ], + dataSource: ServerDataSource(databaseType: databaseType) + ).text + } + + /// What SQL Server's client would send, with the comment lines the dump writes around each statement set aside. + private func batches(of dump: String) async throws -> [String] { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try dump.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + var batches: [String] = [] + for try await (batch, _) in SQLFileParser().parseFile(url: url, encoding: .utf8, grammar: TestGrammar.sqlServer) { + let code = batch + .components(separatedBy: "\n") + .filter { !$0.hasPrefix("--") } + .joined(separator: "\n") + .trimmingCharacters(in: .whitespacesAndNewlines) + batches.append(code) + } + return batches + } + + @Test("The dump source reports SQL Server's GO lines, and no other engine's") + func adapterReportsBatchLines() { + let sqlServer = ExportDataSourceAdapter(driver: MockDatabaseDriver(), databaseType: .mssql) + let mysql = ExportDataSourceAdapter(driver: MockDatabaseDriver(), databaseType: .mysql) + #expect(sqlServer.lexicalFeatures.contains(.batchSeparatorLines)) + #expect(!mysql.lexicalFeatures.contains(.batchSeparatorLines)) + } + + @Test("Every statement of a SQL Server dump is a batch of its own, the view and the routine first in theirs") + func everyStatementIsItsOwnBatch() async throws { + let batches = try await batches(of: try await dump(.mssql)) + + #expect(batches.contains("CREATE VIEW [v_orders] AS SELECT id FROM orders;")) + #expect(batches.contains("CREATE PROCEDURE [p_orders] AS SET NOCOUNT ON; SELECT 1;")) + #expect(batches.contains("CREATE TABLE [orders] ([id] INT IDENTITY(1,1) NOT NULL, [name] NVARCHAR(10) NULL);")) + #expect(batches.contains("SET IDENTITY_INSERT [orders] ON;")) + #expect(batches.contains("SET IDENTITY_INSERT [orders] OFF;")) + #expect(batches.contains { $0.hasPrefix("INSERT INTO [orders]") && $0.hasSuffix(";") }) + #expect(batches.filter { $0.hasPrefix("DROP ") }.count == 3) + } + + @Test("A SQL Server dump ends each statement on a GO line, as SQL Server Management Studio writes one") + func goLinesFollowStatements() async throws { + let dump = try await dump(.mssql) + #expect(dump.contains("SET IDENTITY_INSERT [orders] ON;\nGO\nINSERT INTO [orders]")) + #expect(dump.contains(");\nGO\n\nSET IDENTITY_INSERT [orders] OFF;\nGO\n")) + #expect(dump.contains("CREATE VIEW [v_orders] AS SELECT id FROM orders;\nGO\n")) + } + + @Test("An engine without batches gets no GO line") + func otherEnginesGetNoGoLine() async throws { + let dump = try await dump(.mysql) + #expect(!dump.components(separatedBy: "\n").contains("GO")) + } +} diff --git a/docs/databases/mssql.mdx b/docs/databases/mssql.mdx index 074df3a764..4a89b8fc24 100644 --- a/docs/databases/mssql.mdx +++ b/docs/databases/mssql.mdx @@ -123,6 +123,8 @@ Every result set a batch returns opens in its own result tab. In a batch of plai - The run stops at the first batch that fails, and the error names the line in the editor: `Line 12: Invalid object name 'orders'.` An error raised inside a procedure names the procedure and the line inside it instead. - `Cmd+Enter` with nothing selected runs only the statement at the insertion point. Select the `DECLARE` along with a statement that reads its variable. +**File > Import** runs a `.sql` file the same way, `GO 5` included, and never sends a `GO` line. A failed batch is reported at the line it starts on, and its error names the line in the file: `Failed at line 3. Line 5: Invalid object name 'orders'.` A batch past 64 million characters, about half of what SQL Server takes in one request, ends at its next `;`, so a dump written with no `GO` line still imports. + ## SSL/TLS New connections start on **Preferred**, which encrypts the login and leaves queries and results in plain TCP unless the server forces encryption. Pick **Required (skip verify)** or stricter to encrypt the whole session. diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 98c0b86b0c..c41f58934f 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -45,7 +45,7 @@ SQL exports more than tables. A database holding more than one kind of object gr | Events | `CREATE EVENT` | MySQL, MariaDB | | Privileges | `GRANT`, one principal per row | engines with user management | -An Oracle or Dameng routine or trigger is followed by a `/` line, and a MySQL or MariaDB routine, trigger or event with a body sits in a `DELIMITER` block, the same form a [saved sync script](/features/compare-sync#the-script) uses. Restore those dumps with SQL*Plus, DISQL or the `mysql` client. +An Oracle or Dameng routine or trigger is followed by a `/` line, and a MySQL or MariaDB routine, trigger or event with a body sits in a `DELIMITER` block, the same form a [saved sync script](/features/compare-sync#the-script) uses. Restore those dumps with SQL*Plus, DISQL or the `mysql` client. A SQL Server dump ends every statement with a `GO` line, so each one runs as its own batch in `sqlcmd` and SQL Server Management Studio. A database with only tables lists them flat, with no group to open first. @@ -297,6 +297,8 @@ The extension picks the format, so one command covers all of them. A file no for **UTF-16** reads the byte order from the mark at the start of the file and falls back to big-endian, which is what a file with no mark means. Pick **UTF-16 LE** or **UTF-16 BE** for a file that has no mark and is not big-endian. Latin-1 and Windows-1252 differ over the bytes `0x80` to `0x9F`: a dump written by MySQL keeps its curly quotes, en dashes and euro sign there, so Windows-1252 is the one to pick for it. +On SQL Server a file runs a batch at a time, cut at its `GO` lines the way `sqlcmd` cuts it, so every count and failure here is a batch rather than a statement. See [Scripts and batches](/databases/mssql#scripts-and-batches). + Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. **Save Report…** writes them all to a CSV with a line, a statement and the database's own error per row, so a large import's failures can be sorted and searched rather than scrolled. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. ### Disabling foreign key checks diff --git a/scripts/check-mssql-merge-terminator.sh b/scripts/check-mssql-merge-terminator.sh index d942983e80..b5b9fc5cbe 100755 --- a/scripts/check-mssql-merge-terminator.sh +++ b/scripts/check-mssql-merge-terminator.sh @@ -78,6 +78,7 @@ fi mkdir -p "$WORK/Sources/Check" ln -s "$ROOT/Plugins/MSSQLDriverPlugin/CFreeTDS" "$WORK/CFreeTDS" for source in "$ROOT"/Plugins/MSSQLDriverPlugin/*.swift "$ROOT"/TablePro/Core/Utilities/SQL/SQLFileParser.swift \ + "$ROOT"/TablePro/Core/Utilities/SQL/SQLFileBatchLines.swift \ "$ROOT"/TablePro/Core/Utilities/SQL/SQLChunkDecoder.swift "$ROOT"/TablePro/Core/Utilities/Text/ByteOrderMark.swift; do ln -s "$source" "$WORK/Sources/Check/$(basename "$source")" done @@ -291,8 +292,8 @@ enum Check { return nil } - /// What SQL import reads from the text saved as a file, each statement of which it sends on its own. - static func importedStatements(of text: String) async throws -> [String] { + /// What SQL import reads from the text saved as a file: a batch at a time, each sent whole, the way sqlcmd reads it. + static func importedBatches(of text: String) async throws -> [String] { let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") try text.write(to: url, atomically: true, encoding: .utf8) defer { try? FileManager.default.removeItem(at: url) } @@ -345,12 +346,15 @@ enum Check { "error \(failure ?? "none"), answer \(afterStatements)") _ = try await runBatch(driver, reset) - let imported = try await importedStatements(of: check.text) - let importFailure = await runStatements(driver, imported) + let imported = try await importedBatches(of: check.text) + var importErrors: [PluginBatchError] = [] + for importedBatch in imported { + importErrors += try await runBatch(driver, importedBatch) + } let afterImport = try await answer(driver, check.expectation) - expect(importFailure == nil && afterImport == check.expected, + expect(importErrors.isEmpty && afterImport == check.expected, "\(check.name): imported from a file it runs", - "statements \(imported), error \(importFailure ?? "none"), answer \(afterImport)") + "batches \(imported), errors \(importErrors.map(\.message)), answer \(afterImport)") } guard check.needsTerminator else { continue } diff --git a/scripts/check-mssql-sql-import.sh b/scripts/check-mssql-sql-import.sh new file mode 100755 index 0000000000..8dc3db08b5 --- /dev/null +++ b/scripts/check-mssql-sql-import.sh @@ -0,0 +1,372 @@ +#!/usr/bin/env bash +# +# Check, against a real SQL Server, that a SQL file imported into it runs the way sqlcmd runs it. +# +# sqlcmd and SQL Server Management Studio cut a script into batches at each line holding only GO, send every batch +# whole, and never send the GO line. The import used to split a file at each `;` and send the GO lines on: the server +# ran the statement after a GO, refused the GO itself with Msg 2812 or Msg 102, cut a procedure off at its first inner +# `;`, and dropped every variable before the statement that read it (Msg 137). None of that shows in a unit test, +# because each case depends on what the server does with the text it is sent. +# +# So this builds a harness from the real TablePro/Core/Utilities/SQL/SQLFileParser.swift and the files it reads +# lines and chunks with, the real Plugins/MSSQLDriverPlugin sources, the TableProCore package and the shipped +# Libs/libsybdb.a. Each script is read by SQLFileParser with the SQL Server grammar and every batch it hands out goes +# through MSSQLPluginDriver.executeBatch, as ImportDataSinkAdapter sends it: the first batch that raises an error stops +# the import, with the server's line placed in the file. Then the database is read back. +# +# Usage: +# scripts/check-mssql-sql-import.sh [host] [port] [user] +# +# The password comes from MSSQL_SA_PASSWORD and the database from TP_CHECK_DATABASE (default +# tablepro_sql_import_check, created when missing). With no server listening on host:port, the script starts +# mcr.microsoft.com/azure-sql-edge in Docker as tablepro-mssql-check (or TP_MSSQL_CONTAINER), generating a password +# when none is set, and leaves it running for the next run. Exits 1 when a check fails, 3 when it cannot run. + +set -uo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-14339}" +USER_NAME="${3:-sa}" +PASSWORD="${MSSQL_SA_PASSWORD:-}" +DATABASE="${TP_CHECK_DATABASE:-tablepro_sql_import_check}" +CONTAINER="${TP_MSSQL_CONTAINER:-tablepro-mssql-check}" +IMAGE="mcr.microsoft.com/azure-sql-edge:latest" + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +[ -f "$ROOT/Libs/libsybdb.a" ] || { + echo "not found: Libs/libsybdb.a (run scripts/download-libs.sh)" >&2 + exit 3 +} + +listening() { + nc -z "$HOST" "$PORT" > /dev/null 2>&1 +} + +if ! listening; then + command -v docker > /dev/null 2>&1 || { + echo "no SQL Server at $HOST:$PORT and no docker to start one" >&2 + exit 3 + } + if docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER"; then + echo "starting container $CONTAINER" + docker start "$CONTAINER" > /dev/null || exit 3 + else + if [ -z "$PASSWORD" ]; then + PASSWORD="TpCheck#$(openssl rand -hex 8)" + echo "generated a password for $CONTAINER; export MSSQL_SA_PASSWORD='$PASSWORD' to reuse it" + fi + echo "starting $IMAGE as $CONTAINER on $HOST:$PORT" + docker run -d --name "$CONTAINER" -e ACCEPT_EULA=1 -e "MSSQL_SA_PASSWORD=$PASSWORD" \ + -p "$HOST:$PORT:1433" "$IMAGE" > /dev/null || exit 3 + fi + for _ in $(seq 1 60); do + listening && break + sleep 2 + done +fi + +[ -n "$PASSWORD" ] || { + echo "no password: set MSSQL_SA_PASSWORD for the server at $HOST:$PORT" >&2 + exit 3 +} + +mkdir -p "$WORK/Sources/Check" +ln -s "$ROOT/Plugins/MSSQLDriverPlugin/CFreeTDS" "$WORK/CFreeTDS" +for source in "$ROOT"/Plugins/MSSQLDriverPlugin/*.swift \ + "$ROOT/TablePro/Core/Utilities/SQL/SQLFileParser.swift" \ + "$ROOT/TablePro/Core/Utilities/SQL/SQLFileBatchLines.swift" \ + "$ROOT/TablePro/Core/Utilities/SQL/SQLChunkDecoder.swift" \ + "$ROOT/TablePro/Core/Utilities/Text/ByteOrderMark.swift"; do + ln -s "$source" "$WORK/Sources/Check/$(basename "$source")" +done + +cat > "$WORK/Package.swift" << MANIFEST +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "MSSQLSQLImportCheck", + platforms: [.macOS(.v14)], + dependencies: [.package(path: "$ROOT/Packages/TableProCore")], + targets: [ + .systemLibrary(name: "CFreeTDS", path: "CFreeTDS"), + .executableTarget( + name: "Check", + dependencies: [ + "CFreeTDS", + .product(name: "TableProPluginKit", package: "TableProCore"), + .product(name: "TableProCoreTypes", package: "TableProCore"), + .product(name: "TableProMSSQLCore", package: "TableProCore"), + .product(name: "TableProLogRedaction", package: "TableProCore"), + .product(name: "TableProSQLGrammar", package: "TableProCore"), + ], + path: "Sources/Check", + swiftSettings: [.swiftLanguageMode(.v6)], + linkerSettings: [.unsafeFlags([ + "-L$ROOT/Libs", "-L$ROOT/Libs/dylibs", "-lsybdb", "-lssl.3", "-lcrypto.3", "-liconv", + "-framework", "GSS", "-lcom_err", "-Xlinker", "-rpath", "-Xlinker", "$ROOT/Libs/dylibs", + ])] + ), + ] +) +MANIFEST + +cat > "$WORK/Sources/Check/Check.swift" << 'SWIFT' +import Foundation +import TableProCoreTypes +import TableProPluginKit +import TableProSQLGrammar + +/// The one type SQLFileParser needs from the app beyond the files linked in beside it. +enum DecompressionError: Error { + case decompressFailed + case fileReadFailed(String) +} + +@main +enum Check { + nonisolated(unsafe) static var failures = 0 + + static let grammar = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil).execution + + struct ImportFailure { + let batchLine: Int + let errorLine: Int? + let message: String + } + + static func expect(_ condition: Bool, _ label: String, _ detail: @autoclosure () -> String = "") { + if condition { + print("PASS: \(label)") + } else { + failures += 1 + print("FAIL: \(label) \(detail())") + } + } + + /// The import as the app runs it on SQL Server with Stop and Rollback and no transaction: every run the parser hands + /// out goes to the server whole, and the first that raises an error ends the import. The error's line is the + /// server's line moved onto the file by where the batch starts, which is what `BatchErrorText` does. + static func importScript( + _ text: String, + driver: MSSQLPluginDriver, + parser: SQLFileParser = SQLFileParser() + ) async throws -> (runs: [String], failure: ImportFailure?) { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).sql") + try text.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + var runs: [String] = [] + for try await (batch, line) in parser.parseFile(url: url, encoding: .utf8, grammar: grammar) { + runs.append(batch) + let answer = try await driver.executeBatch(query: batch, rowCap: 1, parameters: nil) + if let error = answer?.errors.first { + let fileLine = error.line.map { line + $0 - 1 } + return (runs, ImportFailure(batchLine: line, errorLine: fileLine, message: error.message)) + } + } + return (runs, nil) + } + + static func scalar(_ sql: String, _ driver: MSSQLPluginDriver) async throws -> String? { + try await driver.execute(query: sql).rows.first?.first?.asText + } + + static func connect(_ database: String) async throws -> MSSQLPluginDriver { + let environment = ProcessInfo.processInfo.environment + let config = DriverConnectionConfig( + host: environment["TP_CHECK_HOST"] ?? "127.0.0.1", + port: Int(environment["TP_CHECK_PORT"] ?? "") ?? 1433, + username: environment["TP_CHECK_USER"] ?? "sa", + password: environment["TP_CHECK_PASSWORD"] ?? "", + database: database + ) + let deadline = Date().addingTimeInterval(120) + while true { + let driver = MSSQLPluginDriver(config: config) + do { + try await driver.connect() + return driver + } catch where Date() < deadline { + try await Task.sleep(nanoseconds: 2_000_000_000) + } + } + } + + static func main() async { + setvbuf(stdout, nil, _IOLBF, 0) + do { + try await run() + } catch { + failures += 1 + print("FAIL: unexpected error \(error)") + } + print(failures == 0 ? "OK: every check passed" : "\(failures) check(s) failed") + exit(failures == 0 ? 0 : 1) + } + + static func run() async throws { + let database = ProcessInfo.processInfo.environment["TP_CHECK_DATABASE"] ?? "tablepro_sql_import_check" + let admin = try await connect("master") + _ = try await admin.execute(query: "IF DB_ID(N'\(database)') IS NULL CREATE DATABASE [\(database)]") + admin.disconnect() + + let driver = try await connect(database) + _ = try await driver.executeBatch(query: """ + IF OBJECT_ID(N'dbo.import_v') IS NOT NULL DROP VIEW dbo.import_v; + IF OBJECT_ID(N'dbo.import_p') IS NOT NULL DROP PROCEDURE dbo.import_p; + IF OBJECT_ID(N'dbo.import_c') IS NOT NULL DROP PROCEDURE dbo.import_c; + IF OBJECT_ID(N'dbo.import_rows') IS NOT NULL DROP TABLE dbo.import_rows; + CREATE TABLE dbo.import_rows (id INT NULL, v NVARCHAR(40) NULL); + EXEC (N'CREATE PROCEDURE dbo.import_p AS SELECT 0;'); + """, rowCap: nil, parameters: nil) + + try await compareScript(driver) + try await ssmsScript(driver) + try await declaredVariable(driver) + try await repeatedBatch(driver) + try await commentsInARoutine(driver) + try await goInsideALiteral(driver) + try await errorLine(driver) + try await dumpWithGoLines(driver) + try await dumpCutAtSemicolons(driver) + driver.disconnect() + } + + static func compareScript(_ driver: MSSQLPluginDriver) async throws { + let script = """ + DROP PROCEDURE [dbo].[import_p]; + GO + CREATE PROCEDURE dbo.import_p AS SET NOCOUNT ON; SELECT 1; + GO + + """ + let result = try await importScript(script, driver: driver) + expect(result.failure == nil, "a Compare script imports", "\(String(describing: result.failure))") + expect(!result.runs.contains { $0.uppercased().contains("\nGO") || $0.uppercased() == "GO" }, + "no GO line reaches the server", "\(result.runs)") + let definition = try await scalar("SELECT OBJECT_DEFINITION(OBJECT_ID(N'dbo.import_p'))", driver) + expect(definition == "CREATE PROCEDURE dbo.import_p AS SET NOCOUNT ON; SELECT 1;", + "the procedure keeps the statement after its inner semicolon", "\(String(describing: definition))") + let answer = try await driver.executeBatch(query: "EXEC dbo.import_p", rowCap: nil, parameters: nil) + expect(answer?.resultSets.first?.rows.first?.first?.asText == "1", "the imported procedure runs") + } + + static func ssmsScript(_ driver: MSSQLPluginDriver) async throws { + let script = """ + DELETE FROM dbo.import_rows + GO + INSERT dbo.import_rows (id, v) VALUES (1, N'one') + INSERT dbo.import_rows (id, v) VALUES (2, N'two') + GO + """ + let result = try await importScript(script, driver: driver) + expect(result.failure == nil, "an SSMS script with no semicolons imports", "\(String(describing: result.failure))") + expect(try await scalar("SELECT COUNT(*) FROM dbo.import_rows", driver) == "2", "both rows arrive") + } + + static func declaredVariable(_ driver: MSSQLPluginDriver) async throws { + let script = """ + DELETE FROM dbo.import_rows; + DECLARE @x INT = 7; + INSERT INTO dbo.import_rows (id) VALUES (@x); + """ + let result = try await importScript(script, driver: driver) + expect(result.failure == nil, "a variable is declared for the statement that reads it", + "\(String(describing: result.failure))") + expect(try await scalar("SELECT MAX(id) FROM dbo.import_rows", driver) == "7", "the variable's value arrives") + } + + static func repeatedBatch(_ driver: MSSQLPluginDriver) async throws { + let script = "DELETE FROM dbo.import_rows\nGO\nINSERT dbo.import_rows (id) VALUES (3)\nGO 3\n" + let result = try await importScript(script, driver: driver) + expect(result.failure == nil, "GO 3 imports", "\(String(describing: result.failure))") + expect(try await scalar("SELECT COUNT(*) FROM dbo.import_rows", driver) == "3", "GO 3 runs its batch three times") + } + + static func commentsInARoutine(_ driver: MSSQLPluginDriver) async throws { + let script = """ + CREATE PROCEDURE dbo.import_c AS + -- keeps this note + SELECT 2; /* and this one */ + GO + """ + let result = try await importScript(script, driver: driver) + expect(result.failure == nil, "a routine with comments imports", "\(String(describing: result.failure))") + let definition = try await scalar("SELECT OBJECT_DEFINITION(OBJECT_ID(N'dbo.import_c'))", driver) ?? "" + expect(definition.contains("-- keeps this note") && definition.contains("/* and this one */"), + "the stored routine keeps the comments written in it", definition) + } + + static func goInsideALiteral(_ driver: MSSQLPluginDriver) async throws { + let script = "DELETE FROM dbo.import_rows;\nINSERT dbo.import_rows (id, v) VALUES (9, N'a\nGO\nb');\nGO\n" + let result = try await importScript(script, driver: driver) + expect(result.failure == nil, "a GO line inside a literal imports", "\(String(describing: result.failure))") + expect(try await scalar("SELECT v FROM dbo.import_rows WHERE id = 9", driver) == "a\nGO\nb", + "the literal keeps its GO line") + } + + static func errorLine(_ driver: MSSQLPluginDriver) async throws { + let script = """ + SELECT 1 + GO + -- the batch below starts on line 3 + + SELECT 2 + SELECT * FROM dbo.no_such_table + GO + SELECT 3 + """ + let result = try await importScript(script, driver: driver) + expect(result.failure?.batchLine == 3, "the failure names the batch's first line", + "\(String(describing: result.failure))") + expect(result.failure?.errorLine == 6, "the server's line lands on the file's line", + "\(String(describing: result.failure))") + expect(result.failure?.message.contains("no_such_table") == true, "the server's own message is kept") + expect(result.runs.count == 2, "the import stops at the batch that failed", "\(result.runs)") + } + + /// The shape TablePro's SQL export writes for SQL Server: every statement a batch of its own, so a view or a routine + /// is first in its batch as SQL Server requires. Written without the GO lines, the same dump fails its view with + /// Msg 111 and runs none of the batch, which is what the export used to write. + static func dumpWithGoLines(_ driver: MSSQLPluginDriver) async throws { + let statements = [ + "IF OBJECT_ID(N'dbo.import_v') IS NOT NULL DROP VIEW dbo.import_v;", + "DELETE FROM dbo.import_rows;", + "INSERT INTO [dbo].[import_rows] ([id], [v]) VALUES (1, N'x'), (2, N'y');", + "CREATE VIEW dbo.import_v AS SELECT id FROM dbo.import_rows;", + ] + let withGo = try await importScript(statements.map { "\($0)\nGO" }.joined(separator: "\n"), driver: driver) + expect(withGo.failure == nil, "a dump with a GO line after each statement imports", + "\(String(describing: withGo.failure))") + expect(try await scalar("SELECT COUNT(*) FROM dbo.import_v", driver) == "2", "the dump's view reads its rows") + + let withoutGo = try await importScript(statements.joined(separator: "\n"), driver: driver) + expect(withoutGo.failure?.message.contains("must be the first statement") == true, + "the same dump with no GO line fails its view as sqlcmd would", "\(String(describing: withoutGo.failure))") + } + + /// A batch past the cut length ends at a semicolon, which is what keeps a dump written with no GO line inside what + /// the server takes in one request. A small cut length stands in for the real one. + static func dumpCutAtSemicolons(_ driver: MSSQLPluginDriver) async throws { + let rows = (1...40).map { "INSERT INTO dbo.import_rows (id, v) VALUES (\($0), N'r;\($0)');" } + let script = (["DELETE FROM dbo.import_rows;"] + rows).joined(separator: "\n") + let result = try await importScript(script, driver: driver, parser: SQLFileParser(batchCutLength: 200)) + expect(result.failure == nil, "a cut dump imports", "\(String(describing: result.failure))") + expect(result.runs.count > 1, "the dump was sent in more than one batch", "\(result.runs.count)") + expect(try await scalar("SELECT COUNT(*) FROM dbo.import_rows", driver) == "40", "every row of a cut dump arrives") + } +} +SWIFT + +export DEVELOPER_DIR="${DEVELOPER_DIR:-$(xcode-select -p)}" +swift build --package-path "$WORK" --scratch-path "$WORK/.build" > "$WORK/build.log" 2>&1 || { + echo "the check failed to build" >&2 + grep -E "error:" "$WORK/build.log" >&2 + exit 3 +} + +TP_CHECK_HOST="$HOST" TP_CHECK_PORT="$PORT" TP_CHECK_USER="$USER_NAME" TP_CHECK_PASSWORD="$PASSWORD" \ + TP_CHECK_DATABASE="$DATABASE" "$WORK/.build/debug/Check"