diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b1999a07..6bc84c988b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -508,8 +508,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Run executing an old sorted query after a column header was clicked while a query ran. - SQL Server scripts refused, or cut to their first result set, over MCP, AppleScript and the AI assistant. - SQL import into SQL Server sending `GO` lines to the server and splitting batches at each semicolon. -- SQL Server dumps failing on their first view, routine or trigger when restored with sqlcmd or imported. +- SQL Server dumps failing on their first view, routine or trigger when restored with sqlcmd or SSMS. - A leading `GO` line sent to SQL Server by MCP and AI assistant tools, and a `GO n` count ignored. +- App hanging for a minute when an import stopped on a failing statement several megabytes long. ### Security diff --git a/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/Plugins/SQLExportPlugin/SQLExportPlugin.swift index 137d9b2553..bb312dbbbc 100644 --- a/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -88,13 +88,6 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi /// says so rather than shipping a dump whose three phases disagree. var exportSpansContainers = false - /// What ends every statement of the dump: a line break, and on an engine whose client runs a - /// script in batches cut at `GO` lines, that line too. SQL Server refuses a view, a routine or a - /// trigger that is not the first statement of its batch, and a routine's body runs to the end - /// of its batch, so each statement is written as a batch of its own, the way SQL Server - /// Management Studio writes a script. - private var statementEnd = "\n" - private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportPlugin") required init() { loadSettings() } @@ -141,7 +134,6 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi exportSpansContainers = false tablesUnorderedByCycle = [] emittedSequenceNames = [] - statementEnd = dataSource.lexicalFeatures.contains(.batchSeparatorLines) ? "\nGO\n" : "\n" /// Read once, because `PluginManager` hands every window the same plugin instance and a /// second window's options pane can write `settings` while this export is still running. A @@ -478,7 +470,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi guard !dropTargets.isEmpty else { return } for object in dropTargets { guard let statement = dropStatement(for: object, dataSource: dataSource) else { continue } - try writer.write(statement + statementEnd) + try writer.write(statement + dataSource.dumpStatementEnd) } try writer.write("\n") } @@ -525,6 +517,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi ) async throws { var emittedTypeNames: Set = [] let structureTables = tables.filter { optionValue($0, at: 0) } + let statementEnd = dataSource.dumpStatementEnd for table in structureTables { do { @@ -597,7 +590,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - try writer.write(dataSource.scriptText(for: ddl) + statementEnd) + try writer.write(dataSource.scriptText(for: ddl) + dataSource.dumpStatementEnd) try writer.write("\n") } catch { ddlFailures.append(sanitizedName) @@ -640,7 +633,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi table: object.name, databaseName: object.databaseName) guard !statements.isEmpty else { return } for statement in statements { - try writer.write(dataSource.scriptText(for: statement) + statementEnd) + try writer.write(dataSource.scriptText(for: statement) + dataSource.dumpStatementEnd) } try writer.write("\n") } catch { @@ -683,7 +676,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - try writer.write(dataSource.scriptText(for: ddl) + statementEnd) + try writer.write(dataSource.scriptText(for: ddl) + dataSource.dumpStatementEnd) try writer.write("\n") } catch { ddlFailures.append(sanitizedName) @@ -717,7 +710,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi principal: principal.name, host: principal.identity) guard !statements.isEmpty else { continue } for statement in statements { - try writer.write(dataSource.scriptText(for: statement) + statementEnd) + try writer.write(dataSource.scriptText(for: statement) + dataSource.dumpStatementEnd) } } catch { let sanitized = PluginExportUtilities.sanitizeForSQLComment(principal.name) @@ -787,7 +780,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let grouped = groupForeignKeysByConstraint(fks) for group in grouped { let alter = renderAddConstraintFK(table: table, group: group, dataSource: dataSource) - try writer.write(alter + statementEnd) + try writer.write(alter + dataSource.dumpStatementEnd) emittedAnything = true } } @@ -802,7 +795,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi for column in columns where column.isIdentity { let setval = renderIdentitySetval( table: table, columnName: column.name, dataSource: dataSource) - try writer.write(setval + statementEnd) + try writer.write(setval + dataSource.dumpStatementEnd) emittedAnything = true } } @@ -838,7 +831,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let statements = try await dataSource.fetchIndexDDL( table: object.name, databaseName: object.databaseName) for statement in statements { - try writer.write(dataSource.scriptText(for: statement) + statementEnd) + try writer.write(dataSource.scriptText(for: statement) + dataSource.dumpStatementEnd) emittedAnything = true } } catch { @@ -959,7 +952,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let needsIdentityInsert = dataSource.databaseTypeId == "SQL Server" && columnInfo.contains(where: \.isIdentity) let identityInsert = needsIdentityInsert - ? SQLExportSessionScope.identityInsert(tableRef: tableRef, statementEnd: statementEnd) + ? SQLExportSessionScope.identityInsert(tableRef: tableRef, statementEnd: dataSource.dumpStatementEnd) : nil if !table.rowScope.isUnrestricted { @@ -1103,7 +1096,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi prefix: rendered.prefix, suffix: rendered.suffix, budget: statementBudget(for: dataSource.databaseTypeId, options: options), - terminator: ";\(statementEnd)\n") + terminator: ";\(dataSource.dumpStatementEnd)\n") return (accumulator, rendered.warning) } diff --git a/Plugins/SQLExportPlugin/SQLExportStatementEnd.swift b/Plugins/SQLExportPlugin/SQLExportStatementEnd.swift new file mode 100644 index 0000000000..a18a9a15e1 --- /dev/null +++ b/Plugins/SQLExportPlugin/SQLExportStatementEnd.swift @@ -0,0 +1,21 @@ +// +// SQLExportStatementEnd.swift +// SQLExportPlugin +// + +import Foundation +import TableProPluginKit + +extension PluginExportDataSource { + /// What ends every statement of a dump of this source: a line break, and on an engine whose client runs a script in + /// batches cut at `GO` lines, that line too. SQL Server refuses a view, a routine or a trigger that is not the first + /// statement of its batch, and a routine's body runs to the end of its batch, so each statement is written as a + /// batch of its own, the way SQL Server Management Studio writes a script. + /// + /// Read off the source each export writes from rather than held by the plugin: `PluginManager` hands every window + /// the same plugin instance, so a second export started while one runs would otherwise change what the first + /// writes from then on. + var dumpStatementEnd: String { + lexicalFeatures.contains(.batchSeparatorLines) ? "\nGO\n" : "\n" + } +} diff --git a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index 8ae4d14c59..cf5b282f9f 100644 --- a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -59,9 +59,10 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { try await execute(statement: statement, line: 1) } - /// A SQL Server file arrives a batch at a time, as sqlcmd reads it, and goes to the server whole: T-SQL scopes a - /// variable, a table variable and a `TRY...CATCH` to one batch, and a routine's body runs to the end of its batch. - /// A driver that cannot send a batch whole runs its statements one by one, as the editor does. + /// A SQL Server file arrives a batch at a time, as sqlcmd reads it, or a statement at a time when it holds no `GO` + /// line, and each goes to the server whole: T-SQL scopes a variable, a table variable and a `TRY...CATCH` to one + /// batch, and a routine's body runs to the end of its batch. A driver that cannot send a batch whole runs its + /// statements one by one, as the editor does. func execute(statement: String, line: Int) async throws { guard grammar.contains(.batchSeparatorLines) else { _ = try await driver.execute(query: statement) diff --git a/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/TablePro/Core/Utilities/SQL/SQLFileParser.swift index adff19bbeb..c69bc6c75f 100644 --- a/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -14,8 +14,8 @@ final class SQLFileParser: Sendable { /// SQL Server takes at most 65,536 packets of 4,096 bytes in one request, about 134 million UTF-16 units of batch /// text: measured on Azure SQL Edge 15, a 133,900,000-unit batch ran and a 133,960,000-unit one closed the /// connection. A batch that reaches half of that ends at its next `;`, which leaves the statement it ends inside - /// as much room again. A dump written with no `GO` line is then sent in pieces the server takes, and held one - /// piece at a time rather than whole. + /// as much room again. A script whose `GO` lines stand that far apart is then sent in pieces the server takes, + /// and held one piece at a time rather than whole. static let defaultBatchCutLength = 67_108_864 private let batchCutLength: Int @@ -165,9 +165,20 @@ final class SQLFileParser: Sendable { /// SQL Server's tools read a script the way sqlcmd does: a line holding only `GO` ends a batch, the batch goes /// to the server whole with its `;` and its comments, and the `GO` line goes nowhere. The comments stay so the /// server's line numbers count the file's lines and a routine keeps the comments written inside it. + /// + /// Only a file that holds a `GO` line is read so. One that holds none was written to run a statement at a + /// time, cut at each `;`: that is how every SQL Server dump TablePro wrote before it wrote `GO` lines reads, + /// and read as one batch it fails on its first view with Msg 111, having run none of it. let readsBatches: Bool let batchCutLength: Int + /// A SQL Server statement goes to the server as a batch of its own, so it keeps the comments written inside it + /// for the same reasons a batch keeps all of its own. The ones before its first code stay out of it. + let keepsStatementComments: Bool + + /// Set once a `GO` line has ended a batch, which is what settles whether a file is read in batches. + var sawBatchSeparator = false + /// Whether only spaces and tabs stand between the last line break and the unit being read, which is where a /// `GO` line can start. Tracked here because the buffer no longer holds the line's start once a chunk is /// consumed. @@ -183,9 +194,9 @@ final class SQLFileParser: Sendable { var unitsBeforeBuffer = 0 /// The statement grammar, for a dialect whose statement can own its `;`: a PL/SQL unit arrives whole with - /// its own `;`. A batch keeps every `;` it holds, a T-SQL `MERGE`'s included, so a grammar read in batches - /// needs none. Every other dialect has always split an import at each `;` and relies on `DELIMITER` or dollar - /// quoting for a routine body, and keeps doing so. + /// its own `;`, and so do a T-SQL `MERGE` and a `BEGIN...END` routine body. A batch keeps every `;` it holds, + /// so a file read in batches needs none. Every other dialect has always split an import at each `;` and + /// relies on `DELIMITER` or dollar quoting for a routine body, and keeps doing so. var boundaries: (any SQLStatementBoundaryTracking)? var word: [unichar] = [] var alternativeQuoteCloser: unichar = 0 @@ -205,20 +216,26 @@ final class SQLFileParser: Sendable { /// the one after it is settled with nothing after it, instead of being left in the buffer and dropped. var atEndOfInput = false - init(grammar: SQLLexicalGrammar, currentStatement: NSMutableString?, batchCutLength: Int) { + init(grammar: SQLLexicalGrammar, currentStatement: NSMutableString?, batchCutLength: Int, readsBatches: Bool) { self.grammar = grammar self.currentStatement = currentStatement - self.readsBatches = grammar.contains(.batchSeparatorLines) + self.readsBatches = readsBatches self.batchCutLength = batchCutLength + self.keepsStatementComments = grammar.contains(.batchSeparatorLines) self.boundaries = !readsBatches && SQLStatementBoundaries.statementsCanOwnTerminator(in: grammar) ? SQLStatementBoundaries.makeTracker(for: grammar) : nil } - /// Whether the block comment being read goes into the statement: a conditional comment is SQL the server runs, - /// and a batch keeps every comment. + /// Whether the comment being read goes into the text sent. + var keepsComments: Bool { + readsBatches || (keepsStatementComments && hasStatementContent) + } + + /// Whether the block comment being read goes into the statement, which a conditional comment always does, + /// being SQL the server runs. var keepsCommentText: Bool { - readsBatches || isConditionalComment + keepsComments || isConditionalComment } } @@ -417,7 +434,7 @@ final class SQLFileParser: Sendable { ) { ctx.state = .inSingleLineComment ctx.boundaries?.observeGap() - if ctx.readsBatches { + if ctx.keepsComments { appendRange(&ctx, from: i, to: i + 2, in: nsBuffer) } i += 2 @@ -429,7 +446,7 @@ final class SQLFileParser: Sendable { || (char == kSlash && nextChar == kSlash && ctx.grammar.contains(.doubleSlashLineComments)) { ctx.state = .inSingleLineComment ctx.boundaries?.observeGap() - if ctx.readsBatches { + if ctx.keepsComments { appendChar(char, to: ctx.currentStatement) } return StepResult(advanced: false, deferred: false) @@ -443,12 +460,12 @@ final class SQLFileParser: Sendable { if ctx.isConditionalComment { (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) + } else { + ctx.boundaries?.observeGap() } if ctx.keepsCommentText { appendChar(char, to: ctx.currentStatement) appendChar(next, to: ctx.currentStatement) - } else { - ctx.boundaries?.observeGap() } i += 2 return StepResult(advanced: true, deferred: false) @@ -774,6 +791,7 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: true) case .separator(let separator): let end = NSMaxRange(separator.range) + ctx.sawBatchSeparator = true yieldBatch(&ctx, repeatCount: separator.repeatCount, nextBatchStart: ctx.unitsBeforeBuffer + end) i = end return StepResult(advanced: true, deferred: false) @@ -819,7 +837,7 @@ final class SQLFileParser: Sendable { appendChar(char, to: ctx.currentStatement) } if char == kSlash, nextChar == kStar, !ctx.isConditionalComment, ctx.grammar.contains(.nestedBlockComments) { - if ctx.readsBatches { + if ctx.keepsComments { appendChar(kStar, to: ctx.currentStatement) } ctx.commentDepth += 1 @@ -969,10 +987,42 @@ final class SQLFileParser: Sendable { }) } + /// How many statements the import runs, a batch ended by `GO 5` counting five times. The runs are added up rather + /// than walked, because a count can reach `Int32.max`. + func countStatements( + url: URL, + encoding: String.Encoding, + grammar: SQLLexicalGrammar + ) async throws -> Int { + let session = ParseSession( + url: url, encoding: encoding, grammar: grammar, countOnly: true, batchCutLength: batchCutLength + ) + var count = 0 + + while let statement = try await session.nextStatement() { + try Task.checkCancellation() + count += statement.repeatCount + } + + return count + } +} + +extension SQLFileParser { + /// How a session cuts its file. + private enum Reading { + /// Settled from the file before its first statement: in batches when it holds a `GO` line and the grammar + /// reads them, else a statement at a time. + case fromFile + /// At the file's `GO` lines, as sqlcmd cuts a script. + case batches + } + private final class ParseSession: @unchecked Sendable { private let url: URL private let encoding: String.Encoding private let grammar: SQLLexicalGrammar + private let batchCutLength: Int private let chunkSize = 65_536 private var fileHandle: FileHandle? @@ -981,18 +1031,29 @@ final class SQLFileParser: Sendable { private var decoder: SQLChunkDecoder private var emitIndex = 0 private var finished = false + private var readingSettled: Bool private var repeating: ParsedStatement? private var remainingRuns = 0 - init(url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar, countOnly: Bool, batchCutLength: Int) { + init( + url: URL, + encoding: String.Encoding, + grammar: SQLLexicalGrammar, + countOnly: Bool, + batchCutLength: Int, + reading: Reading = .fromFile + ) { self.url = url self.encoding = encoding self.grammar = grammar + self.batchCutLength = batchCutLength self.decoder = SQLChunkDecoder(encoding: encoding) + self.readingSettled = reading == .batches self.ctx = ParserContext( grammar: grammar, currentStatement: countOnly ? nil : NSMutableString(), - batchCutLength: batchCutLength + batchCutLength: batchCutLength, + readsBatches: reading == .batches ) } @@ -1024,13 +1085,14 @@ final class SQLFileParser: Sendable { if finished { return nil } - if Task.isCancelled { - finished = true - closeFile() - return nil - } do { + try settleReading() + guard !Task.isCancelled else { + finished = true + closeFile() + return nil + } try advanceOneChunk() } catch { finished = true @@ -1041,6 +1103,40 @@ final class SQLFileParser: Sendable { } } + /// Reads the file for a `GO` line before any of it is handed out, because a batch cannot be put back together + /// from statements already sent. + private func settleReading() throws { + guard !readingSettled else { return } + readingSettled = true + guard grammar.contains(.batchSeparatorLines) else { return } + let scan = ParseSession( + url: url, + encoding: encoding, + grammar: grammar, + countOnly: true, + batchCutLength: batchCutLength, + reading: .batches + ) + guard try scan.findsBatchSeparator() else { return } + ctx = ParserContext( + grammar: grammar, + currentStatement: ctx.currentStatement, + batchCutLength: batchCutLength, + readsBatches: true + ) + } + + /// Whether the file holds a `GO` line, read the way a script is cut into batches, so one inside a literal or a + /// comment does not count. Reading stops at the first. + func findsBatchSeparator() throws -> Bool { + defer { closeFile() } + while !finished, !ctx.sawBatchSeparator, !Task.isCancelled { + try advanceOneChunk() + ctx.collected.removeAll(keepingCapacity: true) + } + return ctx.sawBatchSeparator + } + private func advanceOneChunk() throws { let handle = try openFileIfNeeded() let rawData = handle.readData(ofLength: chunkSize) @@ -1103,7 +1199,7 @@ final class SQLFileParser: Sendable { shouldDefer = result.deferred case .inSingleLineComment: - if ctx.readsBatches { + if ctx.keepsComments { SQLFileParser.appendChar(char, to: ctx.currentStatement) } if char == SQLFileParser.kNewline @@ -1228,24 +1324,4 @@ final class SQLFileParser: Sendable { } } } - - /// How many statements the import runs, a batch ended by `GO 5` counting five times. The runs are added up rather - /// than walked, because a count can reach `Int32.max`. - func countStatements( - url: URL, - encoding: String.Encoding, - grammar: SQLLexicalGrammar - ) async throws -> Int { - let session = ParseSession( - url: url, encoding: encoding, grammar: grammar, countOnly: true, batchCutLength: batchCutLength - ) - var count = 0 - - while let statement = try await session.nextStatement() { - try Task.checkCancellation() - count += statement.repeatCount - } - - return count - } } diff --git a/TablePro/Views/Components/TransferResultAlert.swift b/TablePro/Views/Components/TransferResultAlert.swift index 0d413c18ec..10f4bd45c3 100644 --- a/TablePro/Views/Components/TransferResultAlert.swift +++ b/TablePro/Views/Components/TransferResultAlert.swift @@ -196,28 +196,58 @@ internal enum TransferResultAlert { alert.messageText = String(localized: "Import failed") alert.alertStyle = .critical alert.addButton(withTitle: String(localized: "Done")) + alert.informativeText = importFailureText(for: error) if let pluginError = error as? PluginImportError, - case .statementFailed(let statement, let line, let underlyingError) = pluginError { - alert.informativeText = RevealedText(String( - format: String(localized: "Failed at line %lld. %@"), - Int64(line), - underlyingError.localizedDescription - )).plainText + case .statementFailed(let statement, _, _) = pluginError { alert.accessoryView = TransferReportView( shown: statement, copied: failureReport(for: error) ?? statement ) alert.layout() - } else { - alert.informativeText = RevealedText( - error?.localizedDescription ?? String(localized: "Unknown error") - ).plainText } AlertHelper.present(alert, in: window) { _ in completion() } } + /// The text of a failed import's alert: the line it stopped at and the first of the database's errors. + internal static func importFailureText(for error: (any Error)?) -> String { + guard let pluginError = error as? PluginImportError, + case .statementFailed(_, let line, let underlyingError) = pluginError else { + return RevealedText(error?.localizedDescription ?? String(localized: "Unknown error")).plainText + } + return RevealedText(String( + format: String(localized: "Failed at line %lld. %@"), + Int64(line), + shownErrors(underlyingError.localizedDescription) + )).plainText + } + + /// The most of a failure's errors an alert's text holds, in lines and in UTF-16 units. A SQL Server batch raises + /// one error per statement that failed and the driver keeps up to 1,000, one line each, while an alert grows to fit + /// its text: measured, 1,000 lines made one 54,135 points tall, its Done button far below the screen, and 1,000 + /// units on one line keep it at 423 points. Copy Details carries all of it. + internal static let shownErrorLineLimit = 5 + internal static let shownErrorLengthLimit = 1_000 + + /// The start of `description` the alert has room for, and where the rest is. + internal static func shownErrors(_ description: String) -> String { + let lines = description.components(separatedBy: "\n") + let firstLines = lines.prefix(shownErrorLineLimit).joined(separator: "\n") + let shown = cut(firstLines, toUnits: shownErrorLengthLimit) ?? firstLines + guard shown != description else { return description } + return shown + "\n" + String(localized: "Copy Details copies the rest.") + } + + /// `text` cut to `limit` UTF-16 units on a character boundary, with an ellipsis saying there is more, or nil when + /// it already fits. + internal static func cut(_ text: String, toUnits limit: Int) -> String? { + let source = text as NSString + guard source.length > limit else { return nil } + let end = source.rangeOfComposedCharacterSequence(at: limit).location + return source.substring(to: end) + "\u{2026}" + } + /// The report a failed import puts in its accessory, so the line, the reason and the failing /// statement can be selected and copied together. Returns nil for an error that carries no /// statement, where the alert's own text already says everything there is to say. @@ -310,6 +340,17 @@ internal final class TransferReportView: NSView { private static let textHeight: CGFloat = 140 private static let spacing: CGFloat = 8 + /// The most of a report the box shows. A failed statement can be a whole SQL Server batch, and the text view lays + /// out a paragraph whole on the main thread: measured, a 15 million unit batch holding one 5 million unit line + /// blocked it for 67 seconds and took 2.7 GB, where the first 10,000 units took 0.1 seconds. Copy Details still + /// carries every unit. + internal static let shownLengthLimit = 10_000 + + /// `text` cut to `shownLengthLimit` on a character boundary, with an ellipsis saying there is more. + internal static func shownText(_ text: String) -> String { + TransferResultAlert.cut(text, toUnits: shownLengthLimit) ?? text + } + private let report: String internal convenience init(report: String) { @@ -328,7 +369,7 @@ internal final class TransferReportView: NSView { height: Self.textHeight + Self.spacing + button.frame.height )) - let scroll = TransferResultAlert.scrollingText(RevealedText(shown).plainText) + let scroll = TransferResultAlert.scrollingText(RevealedText(Self.shownText(shown)).plainText) scroll.frame = NSRect( x: 0, y: button.frame.height + Self.spacing, diff --git a/TableProTests/Core/Plugins/SQLServerImportBatchTests.swift b/TableProTests/Core/Plugins/SQLServerImportBatchTests.swift index 0705678f93..2c1490c0a1 100644 --- a/TableProTests/Core/Plugins/SQLServerImportBatchTests.swift +++ b/TableProTests/Core/Plugins/SQLServerImportBatchTests.swift @@ -5,6 +5,7 @@ // A SQL file imported into SQL Server reaches the server a batch at a time, the way sqlcmd sends it. The import used // to split it at every `;` and send each `GO` line on to the server, which ran the statement after a `GO`, refused the // `GO` itself with Msg 2812 or Msg 102, and lost every variable between the statement that declared it and the next. +// A file with no `GO` line still goes a statement at a time, each statement a batch of its own. // import Foundation @@ -180,10 +181,44 @@ struct SQLServerImportBatchTests { } } - @Test("A script with no GO line keeps a declared variable for the statement that reads it") + @Test("A script with a GO line keeps a declared variable for the statement that reads it") func declaredVariableReachesItsReader() async throws { let (sink, driver) = makeSink(declaresBatches: true) - _ = try await runImport("DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);\n", sink: sink) + _ = try await runImport("DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);\nGO\n", sink: sink) #expect(driver.sentBatches.map(\.query) == ["DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);"]) } + + /// The shape of every SQL Server dump TablePro wrote before it wrote `GO` lines. Sent as one batch, SQL Server + /// refuses it with Msg 111 because the view is not the first statement of its batch, and runs none of it. + @Test("A dump with no GO line sends each statement as a batch of its own") + func dumpWithoutGoSendsEachStatement() async throws { + let (sink, driver) = makeSink(declaresBatches: true) + let dump = """ + CREATE TABLE [dbo].[t] ([a] int); + INSERT INTO [dbo].[t] ([a]) VALUES (1), (2); + -- View: v + CREATE VIEW [dbo].[v] AS SELECT a FROM dbo.t; + """ + let result = try await runImport(dump, sink: sink) + #expect(driver.sentBatches.map(\.query) == [ + "CREATE TABLE [dbo].[t] ([a] int)", + "INSERT INTO [dbo].[t] ([a]) VALUES (1), (2)", + "CREATE VIEW [dbo].[v] AS SELECT a FROM dbo.t", + ]) + #expect(result.executedStatements == 3) + } + + @Test("A failed statement in a file with no GO line names the file's own line, past a comment inside it") + func failedStatementReportsTheFileLine() async throws { + let (sink, _) = makeSink(declaresBatches: true) + let script = "SELECT 1;\n-- lead\nSELECT 2\n-- inner\nFROM missing;\nSELECT 3;" + do { + _ = try await runImport(script, sink: sink) + Issue.record("An import whose statement raised an error completed") + } catch let PluginImportError.statementFailed(statement, line, underlying) { + #expect(statement == "SELECT 2\n-- inner\nFROM missing") + #expect(line == 3) + #expect(underlying.localizedDescription == "Line 5: Invalid object name 'missing'.") + } + } } diff --git a/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift b/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift index aba6243b5b..6ccfb7efa4 100644 --- a/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift @@ -6,6 +6,9 @@ // the batch reaches the server whole, and the `GO` line reaches it not at all. The parser streams the file in 64 KiB // chunks, so a `GO` line can be cut in two by a chunk boundary, and the split must not depend on where it falls. // +// A file with no `GO` line at all was written to run a statement at a time, the way TablePro wrote every SQL Server +// dump before it wrote `GO` lines, so it is still cut at each `;`. +// import Foundation @testable import TablePro @@ -90,10 +93,60 @@ struct SQLFileParserBatchTests { ]) } - @Test("A script with no GO line is one batch, so a variable is still declared where it is read") - func scriptWithoutGoIsOneBatch() async throws { - let script = "DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);\n" + /// Read as one batch, this dump fails on its view with Msg 111, "'CREATE VIEW' must be the first statement in a + /// query batch", and runs none of its statements. + @Test("A dump with no GO line runs a statement at a time, and the count agrees") + func dumpWithoutGoRunsStatements() async throws { + let dump = """ + -- TablePro SQL Export + -- Database Type: SQL Server + + CREATE TABLE [dbo].[t] ([a] int NOT NULL); + + INSERT INTO [dbo].[t] ([a]) VALUES (1), (2); + + -- View: v + CREATE VIEW [dbo].[v] AS SELECT a FROM dbo.t; + """ + #expect(try await Self.runs(dump) == [ + Run(statement: "CREATE TABLE [dbo].[t] ([a] int NOT NULL)", line: 4), + Run(statement: "INSERT INTO [dbo].[t] ([a]) VALUES (1), (2)", line: 6), + Run(statement: "CREATE VIEW [dbo].[v] AS SELECT a FROM dbo.t", line: 9), + ]) + #expect(try await Self.count(dump) == 3) + } + + @Test("One GO line makes the whole file a script of batches, so a variable is declared where it is read") + func oneGoLineReadsTheFileInBatches() async throws { + let script = "DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);\nGO\n" #expect(try await Self.statements(script) == ["DECLARE @x INT = 1;\nINSERT INTO t (v) VALUES (@x);"]) + #expect(try await Self.count(script) == 1) + } + + @Test( + "A GO inside a literal, a quoted identifier or a comment does not make a file a script of batches", + arguments: [ + "SELECT 'a\nGO\nb';", + "SELECT N'a\nGO\nb';", + "SELECT [a\nGO\nb] FROM t;", + "SELECT \"a\nGO\nb\" FROM t;", + "SELECT 2 /* a\nGO\n*/;", + "SELECT 2 /* outer /* inner */\nGO\nouter */;", + ] + ) + func goInsideNonCodeLeavesStatements(statement: String) async throws { + let script = "SELECT 1;\n\(statement)" + #expect(try await Self.statements(script) == ["SELECT 1", String(statement.dropLast())]) + #expect(try await Self.count(script) == 2) + } + + @Test("A statement keeps the comments written inside it, so the server's line numbers count the file's lines") + func statementKeepsItsComments() async throws { + let script = "-- lead\nSELECT a--glued\nFROM t;\nCREATE PROCEDURE p AS\nBEGIN\n/* inside */\nSELECT 1;\nEND;" + #expect(try await Self.runs(script) == [ + Run(statement: "SELECT a--glued\nFROM t", line: 2), + Run(statement: "CREATE PROCEDURE p AS\nBEGIN\n/* inside */\nSELECT 1;\nEND", line: 4), + ]) } @Test("GO n runs the batch n times, and the count says so") @@ -121,8 +174,8 @@ struct SQLFileParserBatchTests { arguments: ["GO;", "GO 0", "GOTO done", "go_table", "GO5", "GO /* c */", "SELECT 1 GO", "/* c */ GO"] ) func rejectedLines(line: String) async throws { - let script = "SELECT 1\n\(line)\nSELECT 2" - #expect(try await Self.statements(script) == [script]) + let batch = "SELECT 1\n\(line)\nSELECT 2" + #expect(try await Self.statements(batch + "\nGO") == [batch]) } @Test("GO inside a literal, a quoted identifier or a comment separates nothing, across lines too", arguments: [ @@ -135,13 +188,13 @@ struct SQLFileParserBatchTests { "SELECT 1 -- note\nGO_ON\nSELECT 2", ]) func goInsideNonCode(script: String) async throws { - #expect(try await Self.statements(script) == [script]) + #expect(try await Self.statements(script + "\nGO") == [script]) } @Test("An unterminated block comment swallows every GO line after it, as sqlcmd reads it") func unterminatedCommentSwallowsGo() async throws { - let script = "SELECT 1\n/* open\nGO\nSELECT 2\nGO" - #expect(try await Self.statements(script) == [script]) + let script = "SELECT 1;\nGO\nSELECT 2\n/* open\nGO\nSELECT 3\nGO" + #expect(try await Self.statements(script) == ["SELECT 1;", "SELECT 2\n/* open\nGO\nSELECT 3\nGO"]) } @Test("Comments stay in the batch, so the server's line numbers count the file's lines") @@ -199,7 +252,9 @@ struct SQLFileParserBatchTests { let script = "SELECT 'a\n'GO\nSELECT 2" for boundary in 0..<(script as NSString).length { let padding = "--" + String(repeating: "x", count: Self.chunkSize - boundary - 3) + "\n" - #expect(try await Self.statements(padding + script) == [padding + script], "boundary \(boundary)") + #expect(try await Self.statements(padding + script + "\nGO") == [padding + script], "boundary \(boundary)") + #expect(try await Self.statements(padding + script + ";\nSELECT 3;") == [script, "SELECT 3"], + "boundary \(boundary), no GO line") } } @@ -218,8 +273,8 @@ struct SQLFileParserBatchTests { @Test("A batch below the cut length keeps every statement together") func shortBatchIsNotCut() async throws { - let script = "INSERT t VALUES (1);\nINSERT t VALUES (2);\nINSERT t VALUES (3);" - #expect(try await Self.statements(script) == [script]) + let batch = "INSERT t VALUES (1);\nINSERT t VALUES (2);\nINSERT t VALUES (3);" + #expect(try await Self.statements(batch + "\nGO") == [batch]) } @Test("An engine without batches still splits at each semicolon and drops its comments") diff --git a/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index d0ea3114f5..015d25e008 100644 --- a/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -202,8 +202,8 @@ struct SQLFileParserTests { MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, v) VALUES (s.id, s.v) """ - /// SQL import sends a SQL Server file a batch at a time with its text as written, so a `MERGE` keeps the `;` SQL - /// Server fails it without, with Msg 10713. + /// SQL import sends a SQL Server file with no `GO` line a statement at a time, and SQL Server fails a `MERGE` sent + /// without its `;` with Msg 10713. @Test("SQL Server: a MERGE keeps its ; in the middle of a file and at its end") func sqlServerMergeKeepsItsTerminator() async throws { let sql = """ @@ -213,27 +213,50 @@ struct SQLFileParserTests { TRUNCATE TABLE dbo.\u{6CE8}\u{6587} \(Self.merge); """ - #expect(try await Self.parse(sql, grammar: TestGrammar.sqlServer) == [sql]) + let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) + #expect(stmts == [ + "UPDATE dbo.s SET v = 1", + Self.merge + ";", + "SELECT COUNT(*) FROM dbo.t", + "TRUNCATE TABLE dbo.\u{6CE8}\u{6587}\n" + Self.merge + ";", + ]) } - @Test("SQL Server: a file with no GO line arrives as written, every ; kept", arguments: [ - "SELECT 1\(merge);\nSELECT 1.\(merge);", - "SELECT 1 AS x$merge;\nSELECT 1 AS \u{00E9}merge;\nSELECT a FROM #merge;", - ]) - func sqlServerScriptArrivesAsWritten(sql: String) async throws { - #expect(try await Self.parse(sql, grammar: TestGrammar.sqlServer) == [sql]) + /// SQL Server reads `1MERGE` as a number and a `MERGE`, as the statement scanner does. + @Test("SQL Server: a MERGE glued to a number keeps its ;") + func sqlServerMergeGluedToNumberKeepsItsTerminator() async throws { + let sql = "SELECT 1\(Self.merge);\nSELECT 1.\(Self.merge);" + let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) + #expect(stmts == ["SELECT 1\(Self.merge);", "SELECT 1.\(Self.merge);"]) + } + + @Test("SQL Server: a name that ends in merge is not a MERGE") + func sqlServerNameEndingInMergeDropsTheSeparator() async throws { + let sql = "SELECT 1 AS x$merge;\nSELECT 1 AS \u{00E9}merge;\nSELECT a FROM #merge;" + let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) + #expect(stmts == ["SELECT 1 AS x$merge", "SELECT 1 AS \u{00E9}merge", "SELECT a FROM #merge"]) } @Test("SQL Server: a MERGE whose keyword straddles two read chunks keeps its ;") func sqlServerMergeAcrossChunkBoundary() async throws { let padding = String(repeating: "a", count: 65_523) let sql = "SELECT '\(padding)';\n\(Self.merge);" - #expect(try await Self.parse(sql, grammar: TestGrammar.sqlServer) == [sql]) + let stmts = try await Self.parse(sql, grammar: TestGrammar.sqlServer) + #expect(stmts == ["SELECT '\(padding)'", Self.merge + ";"]) } - /// A T-SQL routine runs to the end of its batch, as sqlcmd sends it, so its body arrives whole. + /// A T-SQL routine has no `DELIMITER` and no dollar quoting, so in a file with no `GO` line only the statement + /// grammar keeps its body whole. @Test("SQL Server: a procedure body arrives whole") func sqlServerProcedureBodyArrivesWhole() async throws { + let procedure = "CREATE PROCEDURE dbo.p AS BEGIN UPDATE dbo.t SET v = 1; \(Self.merge); END" + let stmts = try await Self.parse(procedure + ";\nSELECT 1;", grammar: TestGrammar.sqlServer) + #expect(stmts == [procedure, "SELECT 1"]) + } + + /// A T-SQL routine runs to the end of its batch, as sqlcmd sends it, so its body arrives whole. + @Test("SQL Server: a procedure body arrives whole in its batch") + func sqlServerProcedureBodyArrivesWholeInItsBatch() async throws { let procedure = "CREATE PROCEDURE dbo.p AS BEGIN UPDATE dbo.t SET v = 1; \(Self.merge); END;" let stmts = try await Self.parse(procedure + "\nGO\nSELECT 1;", grammar: TestGrammar.sqlServer) #expect(stmts == [procedure, "SELECT 1;"]) diff --git a/TableProTests/Helpers/SQLExportHarness.swift b/TableProTests/Helpers/SQLExportHarness.swift index 002f31ef15..c42e5e1473 100644 --- a/TableProTests/Helpers/SQLExportHarness.swift +++ b/TableProTests/Helpers/SQLExportHarness.swift @@ -48,6 +48,37 @@ internal actor SQLExportHarness { return (try String(contentsOf: destination, encoding: .utf8), result) } + /// A plugin holding `options` for as long as `body` runs, for a suite that runs several exports on one instance at + /// once, the way two windows do with the one instance `PluginManager` hands every window. + internal func withPlugin( + options: SQLExportOptions = SQLExportOptions(), + _ body: @Sendable (SQLExportPlugin) async throws -> Result + ) async throws -> Result { + let plugin = SQLExportPlugin() + let storedSettings = plugin.settings + plugin.settings = options + defer { plugin.settings = storedSettings } + return try await body(plugin) + } + + /// One export on `plugin`, read back as text. Only for a plugin `withPlugin` hands out, which holds known settings. + internal static func dump( + on plugin: SQLExportPlugin, + tables: [PluginExportTable], + dataSource: any PluginExportDataSource + ) async throws -> String { + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent("\(UUID().uuidString).sql") + defer { try? FileManager.default.removeItem(at: destination) } + _ = try await plugin.export( + tables: tables, + dataSource: dataSource, + destination: destination, + progress: PluginExportProgress(progress: Progress(totalUnitCount: 1)) + ) + return try String(contentsOf: destination, encoding: .utf8) + } + /// The parts a split export wrote, in restore order, and the whole dump as one part when it did /// not split. A split export never writes the name the user chose, so reading that path back /// finds nothing at all. diff --git a/TableProTests/Plugins/SQLExportBatchSeparatorTests.swift b/TableProTests/Plugins/SQLExportBatchSeparatorTests.swift index bb5c1ed84f..ed15445961 100644 --- a/TableProTests/Plugins/SQLExportBatchSeparatorTests.swift +++ b/TableProTests/Plugins/SQLExportBatchSeparatorTests.swift @@ -4,8 +4,8 @@ // // SQL Server's client runs a script in batches cut at lines holding only `GO`, and SQL Server refuses a view, a // routine or a trigger that is not the first statement of its batch with Msg 111. A dump written with only `;` -// between statements is one batch, so its first view failed the whole batch in sqlcmd, in SQL Server Management -// Studio, and in TablePro's own import once that import read SQL Server scripts in batches. +// between statements is one batch, so its first view failed the whole batch in sqlcmd and in SQL Server Management +// Studio. // import Foundation @@ -14,17 +14,47 @@ import TableProPluginKit import TableProSQLGrammar import Testing +/// Holds an export inside its first table definition fetch until the test lets it go, so a second export can run to +/// its end on the same plugin in between. +private actor ExportPause { + private var reached = false + private var arrival: CheckedContinuation? + private var released = false + private var release: CheckedContinuation? + + func hold() async { + reached = true + arrival?.resume() + arrival = nil + guard !released else { return } + await withCheckedContinuation { release = $0 } + } + + func untilReached() async { + guard !reached else { return } + await withCheckedContinuation { arrival = $0 } + } + + func resume() { + released = true + release?.resume() + release = nil + } +} + @Suite("SQL export for an engine that runs scripts in batches") struct SQLExportBatchSeparatorTests { private final class ServerDataSource: PluginExportDataSource, @unchecked Sendable { let databaseTypeId: String let lexicalFeatures: SQLLexicalFeatures private let scriptTextOwner: SQLScriptText + private let pause: ExportPause? - init(databaseType: DatabaseType) { + init(databaseType: DatabaseType, pause: ExportPause? = nil) { self.databaseTypeId = databaseType.rawValue self.lexicalFeatures = databaseType.lexicalGrammar.pluginFeatures self.scriptTextOwner = SQLScriptText(databaseType: databaseType) + self.pause = pause } func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { @@ -45,7 +75,8 @@ struct SQLExportBatchSeparatorTests { } func fetchTableDDL(table: String, databaseName: String) async throws -> String { - "CREATE TABLE [orders] ([id] INT IDENTITY(1,1) NOT NULL, [name] NVARCHAR(10) NULL)" + await pause?.hold() + return "CREATE TABLE [orders] ([id] INT IDENTITY(1,1) NOT NULL, [name] NVARCHAR(10) NULL)" } func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { @@ -88,17 +119,25 @@ struct SQLExportBatchSeparatorTests { kind: kind) } + private var objects: [PluginExportTable] { + [ + object("orders", kind: .table), + object("v_orders", kind: .view), + object("p_orders", kind: .routine), + ] + } + private func dump(_ databaseType: DatabaseType) async throws -> String { try await SQLExportHarness.shared.dump( - tables: [ - object("orders", kind: .table), - object("v_orders", kind: .view), - object("p_orders", kind: .routine), - ], + tables: objects, dataSource: ServerDataSource(databaseType: databaseType) ).text } + private func goLines(in dump: String) -> Int { + dump.components(separatedBy: "\n").filter { $0 == "GO" }.count + } + /// What SQL Server's client would send, with the comment lines the dump writes around each statement set aside. private func batches(of dump: String) async throws -> [String] { let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") @@ -150,4 +189,34 @@ struct SQLExportBatchSeparatorTests { let dump = try await dump(.mysql) #expect(!dump.components(separatedBy: "\n").contains("GO")) } + + /// Every window gets the same plugin instance, so an export can start while another is still writing. Its engine + /// must not change how the first one ends its statements: a SQL Server dump that lost its GO lines partway fails + /// its first view on restore with Msg 111, and a MySQL dump that gained them fails at the first `GO`. + @Test("Two exports on one plugin each end their statements the way their own engine reads them") + func exportsSharingThePluginKeepTheirOwnStatementEnds() async throws { + let expectedGoLines = [ + DatabaseType.mssql: goLines(in: try await dump(.mssql)), + DatabaseType.mysql: goLines(in: try await dump(.mysql)), + ] + #expect(expectedGoLines[.mssql, default: 0] > 0) + + for (heldType, otherType) in [(DatabaseType.mssql, DatabaseType.mysql), (.mysql, .mssql)] { + let pause = ExportPause() + let tables = objects + let (held, other) = try await SQLExportHarness.shared.withPlugin { plugin in + async let held = SQLExportHarness.dump( + on: plugin, tables: tables, dataSource: ServerDataSource(databaseType: heldType, pause: pause) + ) + await pause.untilReached() + let other = try await SQLExportHarness.dump( + on: plugin, tables: tables, dataSource: ServerDataSource(databaseType: otherType) + ) + await pause.resume() + return (try await held, other) + } + #expect(goLines(in: held) == expectedGoLines[heldType], "\(heldType.rawValue) held while another ran") + #expect(goLines(in: other) == expectedGoLines[otherType], "\(otherType.rawValue) run during another") + } + } } diff --git a/TableProTests/Views/TransferFailureReportTests.swift b/TableProTests/Views/TransferFailureReportTests.swift index 47f3d77fa3..15527a9c0a 100644 --- a/TableProTests/Views/TransferFailureReportTests.swift +++ b/TableProTests/Views/TransferFailureReportTests.swift @@ -128,6 +128,90 @@ struct TransferFailureReportTests { #expect(shown == "Line 3: unrecognized token: \"\"") #expect(clipboard.text == report) } + + /// A failed SQL Server batch can run to tens of millions of units, and the box lays its text out on the main + /// thread, so it shows the start of it and leaves the rest to the copy. + @Test("The box shows the start of a long report and the copy carries all of it") + func longReportIsCutInTheBoxButCopiedWhole() { + let original = ClipboardService.shared + defer { ClipboardService.shared = original } + let clipboard = TransferReportClipboard() + ClipboardService.shared = clipboard + + let statement = "INSERT INTO files VALUES (1, 0x" + String(repeating: "A1", count: 500_000) + ");" + let view = TransferReportView(shown: statement, copied: statement) + view.copyReport() + + let shown = view.subviews + .compactMap { ($0 as? NSScrollView)?.documentView as? NSTextView } + .first? + .string ?? "" + #expect((shown as NSString).length == TransferReportView.shownLengthLimit + 1) + #expect(shown.hasSuffix("\u{2026}")) + #expect(statement.hasPrefix(String(shown.dropLast()))) + #expect(clipboard.text == statement) + } + + @Test("A cut report never splits a character in two") + func cutKeepsCharactersWhole() { + let lead = String(repeating: "a", count: TransferReportView.shownLengthLimit - 1) + #expect(TransferReportView.shownText(lead + "\u{1F600}tail") == lead + "\u{2026}") + } + + @Test("A report within the limit is shown as it is") + func shortReportIsShownWhole() { + let report = String(repeating: "b", count: TransferReportView.shownLengthLimit) + #expect(TransferReportView.shownText(report) == report) + } + + /// A SQL Server batch raises one error per statement that failed, up to a thousand, one line each, and an alert + /// grows to fit its text, so the alert names the first few and Copy Details carries them all. + @Test("A failure with many errors names the first few in the alert and copies them all") + func manyErrorsAreCutInTheAlertButCopiedWhole() throws { + let errors = (1...1_000).map { "Line \($0 + 2): Violation of PRIMARY KEY constraint 'PK_t'. Key (\($0))." } + let failure = PluginImportError.statementFailed( + statement: "INSERT t VALUES (1)", + line: 3, + underlyingError: TransferReportStubError(message: errors.joined(separator: "\n")) + ) + + let text = TransferResultAlert.importFailureText(for: failure) + let limit = TransferResultAlert.shownErrorLineLimit + #expect(text.components(separatedBy: "\n").count == limit + 1) + #expect(text.contains(errors[limit - 1])) + #expect(!text.contains(errors[limit])) + + let copied = try #require(TransferResultAlert.failureReport(for: failure)) + #expect(copied.contains(errors[999])) + } + + @Test("An error line too long for the alert is cut there and copied whole") + func longErrorLineIsCutInTheAlertButCopiedWhole() throws { + let reason = "Line 4: Incorrect syntax near '" + String(repeating: "x", count: 100_000) + "'." + let failure = PluginImportError.statementFailed( + statement: "SELECT 1", + line: 3, + underlyingError: TransferReportStubError(message: reason) + ) + + let text = TransferResultAlert.importFailureText(for: failure) + #expect((text as NSString).length < TransferResultAlert.shownErrorLengthLimit + 200) + #expect(text.contains("Line 4: Incorrect syntax near 'xxx")) + + let copied = try #require(TransferResultAlert.failureReport(for: failure)) + #expect(copied.contains(reason)) + } + + @Test("A failure with a few errors names each of them") + func fewErrorsAreShownWhole() { + let failure = PluginImportError.statementFailed( + statement: "SELECT 1", + line: 3, + underlyingError: TransferReportStubError(message: "Line 4: first\nLine 5: second") + ) + let text = TransferResultAlert.importFailureText(for: failure) + #expect(text.hasSuffix("Line 4: first\nLine 5: second")) + } } private struct TransferReportStubError: LocalizedError { diff --git a/docs/databases/mssql.mdx b/docs/databases/mssql.mdx index 4a89b8fc24..cd9777877d 100644 --- a/docs/databases/mssql.mdx +++ b/docs/databases/mssql.mdx @@ -123,7 +123,7 @@ Every result set a batch returns opens in its own result tab. In a batch of plai - The run stops at the first batch that fails, and the error names the line in the editor: `Line 12: Invalid object name 'orders'.` An error raised inside a procedure names the procedure and the line inside it instead. - `Cmd+Enter` with nothing selected runs only the statement at the insertion point. Select the `DECLARE` along with a statement that reads its variable. -**File > Import** runs a `.sql` file the same way, `GO 5` included, and never sends a `GO` line. A failed batch is reported at the line it starts on, and its error names the line in the file: `Failed at line 3. Line 5: Invalid object name 'orders'.` A batch past 64 million characters, about half of what SQL Server takes in one request, ends at its next `;`, so a dump written with no `GO` line still imports. +**File > Import** runs a `.sql` file that holds a `GO` line the same way, `GO 5` included, and never sends a `GO` line. A file with no `GO` line runs one statement at a time, split at each `;`: dumps from TablePro 0.75 and earlier have none and import this way. End such a file with a `GO` line to run it as one batch instead. A failure is reported at the line its batch or statement starts on, and its error names the line in the file: `Failed at line 3. Line 5: Invalid object name 'orders'.` A batch past 64 million characters, about half of what SQL Server takes in one request, ends at its next `;`. ## SSL/TLS @@ -150,6 +150,7 @@ SQL Server connections can also run through the [Cloud SQL Auth Proxy](/connecti - A result set from a batch offers **Fetch All**, and sorts on the server, only when its query can run again on its own. One that reads a variable, or comes from a procedure call or a loop, stays at the row limit and sorts the rows already shown: raise **Row cap** in [Settings](/customization/settings), or run the query by itself. - A batch keeps its first 100 result sets. The statements behind the rest still run, and the status bar says how many result sets were not kept. - With a transaction open under `SET XACT_ABORT ON`, a query cut at the row limit reads its whole result before any rows appear. Stop ends it, and SQL Server then rolls the transaction back. +- Imported from a file with no `GO` line, a procedure, function or trigger whose body is not wrapped in `BEGIN...END` keeps only the first statement of its body, and the statements after it run on their own. Wrap the body in `BEGIN` and `END`, or give every statement of the file a `GO` line after it. ## Troubleshooting diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index c41f58934f..60de8c31c6 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -297,9 +297,9 @@ The extension picks the format, so one command covers all of them. A file no for **UTF-16** reads the byte order from the mark at the start of the file and falls back to big-endian, which is what a file with no mark means. Pick **UTF-16 LE** or **UTF-16 BE** for a file that has no mark and is not big-endian. Latin-1 and Windows-1252 differ over the bytes `0x80` to `0x9F`: a dump written by MySQL keeps its curly quotes, en dashes and euro sign there, so Windows-1252 is the one to pick for it. -On SQL Server a file runs a batch at a time, cut at its `GO` lines the way `sqlcmd` cuts it, so every count and failure here is a batch rather than a statement. See [Scripts and batches](/databases/mssql#scripts-and-batches). +On SQL Server a file that holds a `GO` line runs a batch at a time, cut at its `GO` lines the way `sqlcmd` cuts it, so every count and failure here is a batch rather than a statement. A file with no `GO` line runs a statement at a time. See [Scripts and batches](/databases/mssql#scripts-and-batches). -Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. **Save Report…** writes them all to a CSV with a line, a statement and the database's own error per row, so a large import's failures can be sorted and searched rather than scrolled. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. +Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. **Save Report…** writes them all to a CSV with a line, a statement and the database's own error per row, so a large import's failures can be sorted and searched rather than scrolled. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. The alert shows the first 10,000 characters of a longer statement and the first five errors of a batch that raised more, and **Copy Details** copies all of them. ### Disabling foreign key checks diff --git a/project.yml b/project.yml index b16de9a3c1..f9d0849a32 100644 --- a/project.yml +++ b/project.yml @@ -705,6 +705,7 @@ targets: - Plugins/SQLExportPlugin/SQLExportSessionScope.swift - Plugins/SQLExportPlugin/SQLExportSnapshot.swift - Plugins/SQLExportPlugin/SQLExportStatementBudget.swift + - Plugins/SQLExportPlugin/SQLExportStatementEnd.swift - Plugins/SQLImportPlugin/SQLImportFailure.swift - Plugins/SQLImportPlugin/SQLImportOptions.swift - Plugins/SQLImportPlugin/SQLImportOptionsView.swift diff --git a/scripts/check-mssql-merge-terminator.sh b/scripts/check-mssql-merge-terminator.sh index b5b9fc5cbe..a71e3645de 100755 --- a/scripts/check-mssql-merge-terminator.sh +++ b/scripts/check-mssql-merge-terminator.sh @@ -292,8 +292,9 @@ enum Check { return nil } - /// What SQL import reads from the text saved as a file: a batch at a time, each sent whole, the way sqlcmd reads it. - static func importedBatches(of text: String) async throws -> [String] { + /// What SQL import reads from the text saved as a file, each piece of which it sends whole: a statement at a time + /// for a text with no GO line, as these are, and a batch at a time for one with. + static func imported(_ text: String) async throws -> [String] { let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") try text.write(to: url, atomically: true, encoding: .utf8) defer { try? FileManager.default.removeItem(at: url) } @@ -345,16 +346,18 @@ enum Check { "\(check.name): statement by statement it runs", "error \(failure ?? "none"), answer \(afterStatements)") - _ = try await runBatch(driver, reset) - let imported = try await importedBatches(of: check.text) - var importErrors: [PluginBatchError] = [] - for importedBatch in imported { - importErrors += try await runBatch(driver, importedBatch) + for (file, reading) in [(check.text, "a statement at a time"), (check.text + "\nGO\n", "as a batch")] { + _ = try await runBatch(driver, reset) + let pieces = try await imported(file) + var importErrors: [PluginBatchError] = [] + for piece in pieces { + importErrors += try await runBatch(driver, piece) + } + let afterImport = try await answer(driver, check.expectation) + expect(importErrors.isEmpty && afterImport == check.expected, + "\(check.name): imported from a file \(reading) it runs", + "pieces \(pieces), errors \(importErrors.map(\.message)), answer \(afterImport)") } - let afterImport = try await answer(driver, check.expectation) - expect(importErrors.isEmpty && afterImport == check.expected, - "\(check.name): imported from a file it runs", - "batches \(imported), errors \(importErrors.map(\.message)), answer \(afterImport)") } guard check.needsTerminator else { continue } diff --git a/scripts/check-mssql-sql-import.sh b/scripts/check-mssql-sql-import.sh index 8dc3db08b5..049d613b80 100755 --- a/scripts/check-mssql-sql-import.sh +++ b/scripts/check-mssql-sql-import.sh @@ -5,8 +5,11 @@ # sqlcmd and SQL Server Management Studio cut a script into batches at each line holding only GO, send every batch # whole, and never send the GO line. The import used to split a file at each `;` and send the GO lines on: the server # ran the statement after a GO, refused the GO itself with Msg 2812 or Msg 102, cut a procedure off at its first inner -# `;`, and dropped every variable before the statement that read it (Msg 137). None of that shows in a unit test, -# because each case depends on what the server does with the text it is sent. +# `;`, and dropped every variable before the statement that read it (Msg 137). A file with no GO line at all is still +# cut at each `;`, because that is how every SQL Server dump TablePro wrote before it wrote GO lines reads: sent as one +# batch, it fails on its first view with Msg 111, and on a table the dump drops and recreates wider with Msg 207, and +# runs none of it. None of that shows in a unit test, because each case depends on what the server does with the text +# it is sent. # # So this builds a harness from the real TablePro/Core/Utilities/SQL/SQLFileParser.swift and the files it reads # lines and chunks with, the real Plugins/MSSQLDriverPlugin sources, the TableProCore package and the shipped @@ -219,6 +222,7 @@ enum Check { IF OBJECT_ID(N'dbo.import_p') IS NOT NULL DROP PROCEDURE dbo.import_p; IF OBJECT_ID(N'dbo.import_c') IS NOT NULL DROP PROCEDURE dbo.import_c; IF OBJECT_ID(N'dbo.import_rows') IS NOT NULL DROP TABLE dbo.import_rows; + IF OBJECT_ID(N'dbo.import_wide') IS NOT NULL DROP TABLE dbo.import_wide; CREATE TABLE dbo.import_rows (id INT NULL, v NVARCHAR(40) NULL); EXEC (N'CREATE PROCEDURE dbo.import_p AS SELECT 0;'); """, rowCap: nil, parameters: nil) @@ -226,11 +230,13 @@ enum Check { try await compareScript(driver) try await ssmsScript(driver) try await declaredVariable(driver) + try await declaredVariableWithoutGo(driver) try await repeatedBatch(driver) try await commentsInARoutine(driver) try await goInsideALiteral(driver) try await errorLine(driver) try await dumpWithGoLines(driver) + try await dumpRecreatingAWiderTable(driver) try await dumpCutAtSemicolons(driver) driver.disconnect() } @@ -272,6 +278,7 @@ enum Check { DELETE FROM dbo.import_rows; DECLARE @x INT = 7; INSERT INTO dbo.import_rows (id) VALUES (@x); + GO """ let result = try await importScript(script, driver: driver) expect(result.failure == nil, "a variable is declared for the statement that reads it", @@ -279,6 +286,17 @@ enum Check { expect(try await scalar("SELECT MAX(id) FROM dbo.import_rows", driver) == "7", "the variable's value arrives") } + /// A file with no GO line runs a statement at a time, so a variable declared in one statement is gone by the next. + /// One GO line is what asks for the script's batches. + static func declaredVariableWithoutGo(_ driver: MSSQLPluginDriver) async throws { + let script = "DECLARE @x INT = 7;\nINSERT INTO dbo.import_rows (id) VALUES (@x);\n" + let result = try await importScript(script, driver: driver) + expect(result.runs == ["DECLARE @x INT = 7", "INSERT INTO dbo.import_rows (id) VALUES (@x)"], + "a script with no GO line runs a statement at a time", "\(result.runs)") + expect(result.failure?.message.contains("@x") == true, "the second statement no longer sees the variable", + "\(String(describing: result.failure))") + } + static func repeatedBatch(_ driver: MSSQLPluginDriver) async throws { let script = "DELETE FROM dbo.import_rows\nGO\nINSERT dbo.import_rows (id) VALUES (3)\nGO 3\n" let result = try await importScript(script, driver: driver) @@ -329,8 +347,9 @@ enum Check { } /// The shape TablePro's SQL export writes for SQL Server: every statement a batch of its own, so a view or a routine - /// is first in its batch as SQL Server requires. Written without the GO lines, the same dump fails its view with - /// Msg 111 and runs none of the batch, which is what the export used to write. + /// is first in its batch as SQL Server requires. The export used to write the same dump with no GO line, which + /// sqlcmd sends as one batch and fails on its view with Msg 111, running none of it. The import still reads that + /// dump a statement at a time. static func dumpWithGoLines(_ driver: MSSQLPluginDriver) async throws { let statements = [ "IF OBJECT_ID(N'dbo.import_v') IS NOT NULL DROP VIEW dbo.import_v;", @@ -343,16 +362,39 @@ enum Check { "\(String(describing: withGo.failure))") expect(try await scalar("SELECT COUNT(*) FROM dbo.import_v", driver) == "2", "the dump's view reads its rows") + _ = try await driver.executeBatch(query: "DROP VIEW dbo.import_v;", rowCap: nil, parameters: nil) let withoutGo = try await importScript(statements.joined(separator: "\n"), driver: driver) - expect(withoutGo.failure?.message.contains("must be the first statement") == true, - "the same dump with no GO line fails its view as sqlcmd would", "\(String(describing: withoutGo.failure))") + expect(withoutGo.failure == nil, "the same dump with no GO line imports a statement at a time", + "\(String(describing: withoutGo.failure))") + expect(withoutGo.runs.count == statements.count, "each of its statements is a batch of its own", + "\(withoutGo.runs)") + expect(try await scalar("SELECT COUNT(*) FROM sys.views WHERE name = N'import_v'", driver) == "1", + "the view of a dump with no GO line is created") + } + + /// A dump written with its drop option restores over the database it came from. Sent as one batch, the INSERT is + /// compiled against the table as it stands before the batch runs, which has only the column it had then, and SQL + /// Server refuses the whole batch with Msg 207. + static func dumpRecreatingAWiderTable(_ driver: MSSQLPluginDriver) async throws { + _ = try await driver.executeBatch(query: "CREATE TABLE dbo.import_wide (a INT);", rowCap: nil, parameters: nil) + let dump = """ + DROP TABLE IF EXISTS [dbo].[import_wide]; + CREATE TABLE [dbo].[import_wide] ([a] int, [b] int); + INSERT INTO [dbo].[import_wide] ([a], [b]) VALUES (1, 2); + """ + let result = try await importScript(dump, driver: driver) + expect(result.failure == nil, "a dump that recreates a table wider imports", "\(String(describing: result.failure))") + let shape = try await scalar( + "SELECT CONCAT(COUNT(*), N':', COL_LENGTH(N'dbo.import_wide', N'b')) FROM dbo.import_wide", driver + ) + expect(shape == "1:4", "the recreated table has its new column and its row", "\(String(describing: shape))") } - /// A batch past the cut length ends at a semicolon, which is what keeps a dump written with no GO line inside what - /// the server takes in one request. A small cut length stands in for the real one. + /// A batch past the cut length ends at a semicolon, which is what keeps a script whose GO lines stand far apart + /// inside what the server takes in one request. A small cut length stands in for the real one. static func dumpCutAtSemicolons(_ driver: MSSQLPluginDriver) async throws { let rows = (1...40).map { "INSERT INTO dbo.import_rows (id, v) VALUES (\($0), N'r;\($0)');" } - let script = (["DELETE FROM dbo.import_rows;"] + rows).joined(separator: "\n") + let script = (["DELETE FROM dbo.import_rows;"] + rows + ["GO"]).joined(separator: "\n") let result = try await importScript(script, driver: driver, parser: SQLFileParser(batchCutLength: 200)) expect(result.failure == nil, "a cut dump imports", "\(String(describing: result.failure))") expect(result.runs.count > 1, "the dump was sent in more than one batch", "\(result.runs.count)")