fix(plugins): run SQL Server imports as GO batches and end each statement of a SQL Server dump with GO

CHANGELOG.md

@@ -506,6 +506,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Previous run's column headings left over an empty grid under the error of a script that failed partway.
 - Row numbers of a longer previous result left beside the rows of a shorter one.
 - Run executing an old sorted query after a column header was clicked while a query ran.
+- SQL import into SQL Server sending `GO` lines to the server and splitting batches at each semicolon.
+- SQL Server dumps failing on their first view, routine or trigger when restored with sqlcmd or imported.
 
 ### Security
 

Packages/TableProCore/Sources/TableProSQLGrammar/SQLBatchSeparator.swift

@@ -39,11 +39,26 @@ public struct SQLBatchSeparator: Sendable, Equatable {
         in text: NSString,
         length: Int,
         grammar: SQLLexicalGrammar
+    ) -> SQLBatchSeparator? {
+        guard startsLine(text, at: offset) else { return nil }
+        return line(startingAt: offset, in: text, length: length, grammar: grammar)
+    }
+
+    /// The separator whose `GO` starts at `offset`, for a reader that has already seen that nothing but spaces and
+    /// tabs stand before it on its line, such as one reading a file a chunk at a time whose buffer no longer holds
+    /// the start of the line.
+    ///
+    /// `text` has to hold the whole line, its line break included, unless the line ends the text: the rest of the
+    /// line is what decides, and a line cut short at `length` reads as ending there.
+    public static func line(
+        startingAt offset: Int,
+        in text: NSString,
+        length: Int,
+        grammar: SQLLexicalGrammar
     ) -> SQLBatchSeparator? {
         guard offset + 1 < length,
               isG(text.character(at: offset)),
-              isO(text.character(at: offset + 1)),
-              startsLine(text, at: offset)
+              isO(text.character(at: offset + 1))
         else {
             return nil
         }

Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBatchSeparatorTests.swift

@@ -83,6 +83,30 @@ struct SQLBatchSeparatorTests {
         #expect(separators(text).isEmpty)
     }
 
+    @Test(
+        "A reader that tracked the line start reads each line as the scanner does",
+        arguments: [
+            "GO", "go", "GO   ", "GO -- batch one", "GO--glued", "GO 2 -- twice", "GO\t7", "GO 05", "GO 2147483647",
+            "GO;", "GO 0", "GO -1", "GO x", "GO 2147483648", "GOTO done", "go_table", "GO5", "GO /* c */", "GO 2 3",
+        ]
+    )
+    func lineStartingAtAgreesWithTheScanner(line: String) {
+        let text = "SELECT 1\n\(line)\nSELECT 2"
+        let lineStart = 9
+        let buffer = "\(line)\nSELECT 2" as NSString
+        let read = SQLBatchSeparator.line(startingAt: 0, in: buffer, length: buffer.length, grammar: Self.sqlServer)
+        let scanned = separators(text).first
+        #expect(read?.repeatCount == scanned?.repeatCount)
+        #expect(read.map { NSRange(location: $0.range.location + lineStart, length: $0.range.length) } == scanned?.range)
+    }
+
+    @Test("The caller vouches for the line start, so text before the GO is not read")
+    func lineStartingAtTrustsTheCaller() {
+        let buffer = "  GO 3\nSELECT 2" as NSString
+        let read = SQLBatchSeparator.line(startingAt: 2, in: buffer, length: buffer.length, grammar: Self.sqlServer)
+        #expect(read == SQLBatchSeparator(range: NSRange(location: 2, length: 4), repeatCount: 3))
+    }
+
     @Test("GO after code on the same line is not a separator")
     func goMustStartTheLine() {
         #expect(separators("SELECT 1 GO\nSELECT 2").isEmpty)

Plugins/SQLExportPlugin/SQLExportPlugin.swift

@@ -88,6 +88,13 @@
     /// says so rather than shipping a dump whose three phases disagree.
     var exportSpansContainers = false
 
+    /// What ends every statement of the dump: a line break, and on an engine whose client runs a
+    /// script in batches cut at `GO` lines, that line too. SQL Server refuses a view, a routine or a
+    /// trigger that is not the first statement of its batch, and a routine's body runs to the end
+    /// of its batch, so each statement is written as a batch of its own, the way SQL Server
+    /// Management Studio writes a script.
+    private var statementEnd = "\n"
+
     private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportPlugin")
 
     required init() { loadSettings() }
@@ -134,6 +141,7 @@
         exportSpansContainers = false
         tablesUnorderedByCycle = []
         emittedSequenceNames = []
+        statementEnd = dataSource.lexicalFeatures.contains(.batchSeparatorLines) ? "\nGO\n" : "\n"
 
         /// Read once, because `PluginManager` hands every window the same plugin instance and a
         /// second window's options pane can write `settings` while this export is still running. A
@@ -470,7 +478,7 @@
         guard !dropTargets.isEmpty else { return }
         for object in dropTargets {
             guard let statement = dropStatement(for: object, dataSource: dataSource) else { continue }
-            try writer.write("\(statement)\n")
+            try writer.write(statement + statementEnd)
         }
         try writer.write("\n")
     }
@@ -527,9 +535,9 @@
                     let quotedName = "\"\(seq.name.replacingOccurrences(of: "\"", with: "\"\""))\""
                     if optionValue(table, at: 1) {
                         try writer.write(
-                            "DROP SEQUENCE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\n")
+                            "DROP SEQUENCE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\(statementEnd)")
                     }
-                    try writer.write("\(seq.ddl)\n\n")
+                    try writer.write("\(seq.ddl)\(statementEnd)\n")
                 }
             } catch {
                 let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(table.name)
@@ -547,10 +555,11 @@
                     let quotedName = "\"\(enumType.name.replacingOccurrences(of: "\"", with: "\"\""))\""
                     if optionValue(table, at: 1) {
                         try writer.write(
-                            "DROP TYPE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\n")
+                            "DROP TYPE IF EXISTS \(quotedName)\(cascadeClause(dataSource));\(statementEnd)")
                     }
                     let quotedLabels = enumType.labels.map { "'\(dataSource.escapeStringLiteral($0))'" }
-                    try writer.write("CREATE TYPE \(quotedName) AS ENUM (\(quotedLabels.joined(separator: ", ")));\n\n")
+                    let labels = quotedLabels.joined(separator: ", ")
+                    try writer.write("CREATE TYPE \(quotedName) AS ENUM (\(labels));\(statementEnd)\n")
                 }
             } catch {
                 let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(table.name)
@@ -588,8 +597,8 @@
                 guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
                     throw SQLExportObjectError.emptyDefinition
                 }
-                try writer.write(dataSource.scriptText(for: ddl))
-                try writer.write("\n\n")
+                try writer.write(dataSource.scriptText(for: ddl) + statementEnd)
+                try writer.write("\n")
             } catch {
                 ddlFailures.append(sanitizedName)
                 let ddlWarning = "Warning: failed to fetch DDL for table \(sanitizedName): \(error)"
@@ -631,7 +640,7 @@
                 table: object.name, databaseName: object.databaseName)
             guard !statements.isEmpty else { return }
             for statement in statements {
-                try writer.write("\(dataSource.scriptText(for: statement))\n")
+                try writer.write(dataSource.scriptText(for: statement) + statementEnd)
             }
             try writer.write("\n")
         } catch {
@@ -674,8 +683,8 @@
                 guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
                     throw SQLExportObjectError.emptyDefinition
                 }
-                try writer.write(dataSource.scriptText(for: ddl))
-                try writer.write("\n\n")
+                try writer.write(dataSource.scriptText(for: ddl) + statementEnd)
+                try writer.write("\n")
             } catch {
                 ddlFailures.append(sanitizedName)
                 Self.logger.warning("Failed to fetch DDL for \(sanitizedName): \(error)")
@@ -708,7 +717,7 @@
                     principal: principal.name, host: principal.identity)
                 guard !statements.isEmpty else { continue }
                 for statement in statements {
-                    try writer.write("\(dataSource.scriptText(for: statement))\n")
+                    try writer.write(dataSource.scriptText(for: statement) + statementEnd)
                 }
             } catch {
                 let sanitized = PluginExportUtilities.sanitizeForSQLComment(principal.name)
@@ -778,7 +787,7 @@
                 let grouped = groupForeignKeysByConstraint(fks)
                 for group in grouped {
                     let alter = renderAddConstraintFK(table: table, group: group, dataSource: dataSource)
-                    try writer.write("\(alter)\n")
+                    try writer.write(alter + statementEnd)
                     emittedAnything = true
                 }
             }
@@ -793,7 +802,7 @@
                 for column in columns where column.isIdentity {
                     let setval = renderIdentitySetval(
                         table: table, columnName: column.name, dataSource: dataSource)
-                    try writer.write("\(setval)\n")
+                    try writer.write(setval + statementEnd)
                     emittedAnything = true
                 }
             }
@@ -829,7 +838,7 @@
                 let statements = try await dataSource.fetchIndexDDL(
                     table: object.name, databaseName: object.databaseName)
                 for statement in statements {
-                    try writer.write("\(dataSource.scriptText(for: statement))\n")
+                    try writer.write(dataSource.scriptText(for: statement) + statementEnd)
                     emittedAnything = true
                 }
             } catch {
@@ -950,7 +959,7 @@
         let needsIdentityInsert = dataSource.databaseTypeId == "SQL Server"
             && columnInfo.contains(where: \.isIdentity)
         let identityInsert = needsIdentityInsert
-            ? SQLExportSessionScope.identityInsert(tableRef: tableRef)
+            ? SQLExportSessionScope.identityInsert(tableRef: tableRef, statementEnd: statementEnd)
             : nil
 
         if !table.rowScope.isUnrestricted {
@@ -1009,7 +1018,7 @@
                 if let encoder { tally.unrepresentableValues += encoder.unrepresentableValues.total }
                 let built = SQLExportRowValueEncoder(
                     columns: header.columns,
                     columnTypeNames: header.columnTypeNames ?? [],
                     excludedColumnNames: generatedColumnNames,
                     databaseTypeId: dataSource.databaseTypeId,
                     escapeStringLiteral: dataSource.escapeStringLiteral
@@ -1093,7 +1102,8 @@
         let accumulator = SQLExportStatementAccumulator(
             prefix: rendered.prefix,
             suffix: rendered.suffix,
-            budget: statementBudget(for: dataSource.databaseTypeId, options: options))
+            budget: statementBudget(for: dataSource.databaseTypeId, options: options),
+            terminator: ";\(statementEnd)\n")
         return (accumulator, rendered.warning)
     }
 

Plugins/SQLExportPlugin/SQLExportSessionScope.swift

@@ -21,9 +21,12 @@ internal struct SQLExportSessionScope: Equatable {
     /// plain statements, one `ON` per table and one `OFF` after the stream, a rotation between them
     /// left part N+1 answering every row with "Cannot insert explicit value for identity column"
     /// while the export reported success (#2533).
-    internal static func identityInsert(tableRef: String) -> SQLExportSessionScope {
+    ///
+    /// `statementEnd` is what ends every statement of the dump, a `GO` line included where the dump
+    /// carries one, so the pair are statements like any other.
+    internal static func identityInsert(tableRef: String, statementEnd: String) -> SQLExportSessionScope {
         SQLExportSessionScope(
-            opener: "SET IDENTITY_INSERT \(tableRef) ON;\n",
-            closer: "SET IDENTITY_INSERT \(tableRef) OFF;\n")
+            opener: "SET IDENTITY_INSERT \(tableRef) ON;\(statementEnd)",
+            closer: "SET IDENTITY_INSERT \(tableRef) OFF;\(statementEnd)")
     }
 }

Plugins/SQLExportPlugin/SQLExportStatementBudget.swift

@@ -70,17 +70,17 @@ internal struct SQLExportStatementTally: Equatable {
 internal final class SQLExportStatementAccumulator {
     /// What separates two rows, and what terminates a statement. Both are counted as the UTF-8 bytes
     /// they are, because the file is written as UTF-8 whatever encoding its prologue declares to the
-    /// server.
+    /// server. A terminator can carry a `GO` line, which ends each statement's batch on SQL Server.
     private static let rowSeparator = ",\n"
-    private static let terminator = ";\n\n"
     private static let rowSeparatorBytes = 2
-    private static let terminatorBytes = 3
 
     private let prefix: String
     private let suffix: String
+    private let terminator: String
     private let budget: SQLExportStatementBudget
     private let prefixBytes: Int
     private let suffixBytes: Int
+    private let terminatorBytes: Int
 
     private var rows: [String] = []
     private var bodyBytes = 0
@@ -90,12 +90,19 @@ internal final class SQLExportStatementAccumulator {
     internal private(set) var oversizedRowCount = 0
     internal private(set) var statementCount = 0
 
-    internal init(prefix: String, suffix: String, budget: SQLExportStatementBudget) {
+    internal init(
+        prefix: String,
+        suffix: String,
+        budget: SQLExportStatementBudget,
+        terminator: String = ";\n\n"
+    ) {
         self.prefix = prefix
         self.suffix = suffix
+        self.terminator = terminator
         self.budget = budget
         prefixBytes = prefix.utf8.count
         suffixBytes = suffix.utf8.count
+        terminatorBytes = terminator.utf8.count
     }
 
     /// What this accumulator wrote, against the limit it was built with rather than whatever the
@@ -112,7 +119,7 @@ internal final class SQLExportStatementAccumulator {
     /// trailing clause is not small: `ON DUPLICATE KEY UPDATE` over a wide table, or a PostgreSQL
     /// `ON CONFLICT ... DO UPDATE SET`, runs to hundreds of bytes that a budget counting only values
     /// would spend twice.
-    private var envelopeBytes: Int { prefixBytes + suffixBytes + Self.terminatorBytes }
+    private var envelopeBytes: Int { prefixBytes + suffixBytes + terminatorBytes }
 
     /// Takes one rendered row and hands back the statement it closed, if it closed one.
     internal func append(_ renderedRow: String) -> String? {
@@ -142,7 +149,7 @@ internal final class SQLExportStatementAccumulator {
     /// Closes whatever is held, and answers nil when nothing is.
     internal func finish() -> String? {
         guard !rows.isEmpty else { return nil }
-        let statement = prefix + rows.joined(separator: Self.rowSeparator) + suffix + Self.terminator
+        let statement = prefix + rows.joined(separator: Self.rowSeparator) + suffix + terminator
         let statementBytes = envelopeBytes + bodyBytes
         if statementBytes > largestStatementBytes {
             largestStatementBytes = statementBytes

Plugins/SQLImportPlugin/SQLImportPlugin.swift

@@ -73,7 +73,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi
                 try progress.checkCancellation()
 
                 do {
-                    try await sink.execute(statement: statement)
+                    try await sink.execute(statement: statement, line: lineNumber)
                     executedCount += 1
                     progress.incrementStatement()
                 } catch {

Plugins/TableProPluginKit/PluginExportDataSource.swift

@@ -40,6 +40,12 @@ public protocol PluginExportDataSource: AnyObject, Sendable {
     /// does not already end in one.
     func scriptText(for ddl: String) -> String
 
+    /// How this engine's own client reads a script, so a dump is written the way it is read back. SQL Server's
+    /// client ends a batch at a line holding only `GO` (``SQLLexicalFeatures/batchSeparatorLines``) and refuses a
+    /// view, a routine or a trigger that is not the first statement of its batch. Empty by default, which writes a
+    /// script of `;`-terminated statements.
+    var lexicalFeatures: SQLLexicalFeatures { get }
+
     /// The GRANT statements that recreate one principal's privileges, rendered by the engine's own
     /// grant builder. `host` is the MySQL-style host part, which is what separates two principals
     /// that share a name. Empty on an engine with no principal management.
@@ -86,6 +92,8 @@ public extension PluginExportDataSource {
         ddl.hasSuffix(";") ? ddl : ddl + ";"
     }
 
+    var lexicalFeatures: SQLLexicalFeatures { [] }
+
     func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { [] }
 
     var supportsCascadeDrop: Bool { false }

Plugins/TableProPluginKit/PluginImportDataSink.swift

@@ -9,6 +9,12 @@ public protocol PluginImportDataSink: AnyObject, Sendable {
     var databaseTypeId: String { get }
     var targetTable: String? { get }
     func execute(statement: String) async throws
+
+    /// Runs one statement read from a file that starts on the file's `line`, so an error the database places inside
+    /// the statement can be placed in the file. On SQL Server the statement is a whole batch, and the server counts
+    /// its lines from the batch's first.
+    func execute(statement: String, line: Int) async throws
+
     func insertRow(_ values: [String: PluginCellValue]) async throws
     func insertRows(_ rows: [[String: PluginCellValue]]) async throws
     func deleteAllRowsFromTargetTable() async throws
@@ -21,6 +27,9 @@ public protocol PluginImportDataSink: AnyObject, Sendable {
 
 public extension PluginImportDataSink {
     var targetTable: String? { nil }
+    func execute(statement: String, line: Int) async throws {
+        try await execute(statement: statement)
+    }
     func insertRow(_ values: [String: PluginCellValue]) async throws {
         throw PluginImportError.importFailed("Row-based import is not supported by this connection")
     }

TablePro/Core/Plugins/ExportDataSourceAdapter.swift

@@ -6,6 +6,7 @@
 import Foundation
 import os
 import TableProPluginKit
+import TableProSQLGrammar
 
 final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable {
     let databaseTypeId: String
@@ -20,13 +21,15 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable
     /// construction, on the main actor, because the registry lives there and this is asked for from
     /// the export plugin's own thread.
     let supportsCascadeDrop: Bool
+    let lexicalFeatures: SQLLexicalFeatures
     private let implicitSchemaName: String?
     private let pagination: PaginationCapability
     private let cappedTables = OSAllocatedUnfairLock<[String]>(initialState: [])
 
     init(driver: DatabaseDriver, databaseType: DatabaseType) {
         let snapshot = PluginMetadataRegistry.shared.snapshot(for: databaseType)
         self.supportsCascadeDrop = snapshot?.capabilities.supportsCascadeDrop ?? false
+        self.lexicalFeatures = databaseType.lexicalGrammar.pluginFeatures
         self.implicitSchemaName = snapshot?.schema.implicitSchemaName
         self.pagination = PaginationCapability.of(databaseType)
         self.driver = driver