fix(tabs): reload only the tabs showing a changed table, never interrupt one holding edits, and read SQLite result columns after the first step

CHANGELOG.md

@@ -524,6 +524,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Diagram Fit to Window stopping a scroll bar's width short when scroll bars are always shown.
 - No Executing indicator or Stop button for a query tab with no result grid, in Output mode or on a query plan.
 - Executing indicator and Stop button carried over for a moment onto the query tab switched to.
+- Every window's front tab reloading, and asking to discard its edits, after a row import, a new table or a structure change.
+- Tabs showing old rows, columns, DDL or triggers after a save, import, structure change or materialized view refresh.
+- Tables with hidden columns querying a dropped column, or leaving out a new one, after a SQL file import or a structure change.
+- First SQLite or libSQL query after a structure save showing the table's old columns, without the new column's values.
+- Edits after a structure change using the table's old primary key, defaults and generated columns.
 
 ### Security
 

Packages/TableProCore/Sources/TableProSQLiteCore/SQLiteResultColumns.swift

@@ -0,0 +1,35 @@
+import CSQLite
+
+/// The columns a statement returns, read once its first step has run.
+///
+/// `sqlite3_prepare_v2` compiles against the schema this connection last read and does not look at
+/// the file. The first `sqlite3_step` compares the schema cookie and, when another connection has
+/// changed the table since, prepares the statement again with the new columns. Read before that
+/// step, a `SELECT *` on the session connection right after an `ALTER TABLE` run on a pooled one
+/// named the old columns and dropped the new column's values from every row. The structure editor
+/// alters on a pooled connection, so that was the first reload after every structure save.
+///
+/// Both plugins that link their own SQLite share this, so it compiles against that library's header
+/// and never against the SDK's `SQLite3` module, whose `link "sqlite3"` would ask the linker for
+/// macOS's build too.
+public enum SQLiteResultColumns {
+    public struct FirstStep: Sendable {
+        public let result: Int32
+        public let names: [String]
+        public let typeNames: [String]
+
+        public var count: Int32 { Int32(names.count) }
+    }
+
+    public static func stepFirst(_ statement: OpaquePointer?) -> FirstStep {
+        let result = sqlite3_step(statement)
+        let count = sqlite3_column_count(statement)
+        var names: [String] = []
+        var typeNames: [String] = []
+        for index in 0..<count {
+            names.append(sqlite3_column_name(statement, index).map { String(cString: $0) } ?? "column_\(index)")
+            typeNames.append(sqlite3_column_decltype(statement, index).map { String(cString: $0) } ?? "")
+        }
+        return FirstStep(result: result, names: names, typeNames: typeNames)
+    }
+}

Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift

@@ -96,20 +96,23 @@ actor SQLiteLocalBackend {
 
         try bind(parameters, to: statement, db: db)
 
-        let columnCount = sqlite3_column_count(statement)
-        let columns = columnNames(of: statement, count: columnCount)
-        let columnTypeNames = columnDeclaredTypes(of: statement, count: columnCount)
+        let firstStep = SQLiteResultColumns.stepFirst(statement)
+        let columnCount = firstStep.count
+        let columns = firstStep.names
+        let columnTypeNames = firstStep.typeNames
 
         var rows: [[PluginCellValue]] = []
         var rowsAffected = 0
         var truncated = false
 
-        while sqlite3_step(statement) == SQLITE_ROW {
+        var stepResult = firstStep.result
+        while stepResult == SQLITE_ROW {
             if rows.count >= PluginRowLimits.emergencyMax {
                 truncated = true
                 break
             }
             rows.append(rowValues(of: statement, count: columnCount))
+            stepResult = sqlite3_step(statement)
         }
 
         if columns.isEmpty {
@@ -142,18 +145,20 @@ actor SQLiteLocalBackend {
             throw LibSQLError(message: errorMessage)
         }
 
-        let columnCount = sqlite3_column_count(statement)
+        let firstStep = SQLiteResultColumns.stepFirst(statement)
+        let columnCount = firstStep.count
         continuation.yield(.header(PluginStreamHeader(
-            columns: columnNames(of: statement, count: columnCount),
-            columnTypeNames: columnDeclaredTypes(of: statement, count: columnCount),
+            columns: firstStep.names,
+            columnTypeNames: firstStep.typeNames,
             estimatedRowCount: nil
         )))
 
         let batchSize = 5_000
         var batch: [PluginRow] = []
         batch.reserveCapacity(batchSize)
 
-        while sqlite3_step(statement) == SQLITE_ROW {
+        var stepResult = firstStep.result
+        while stepResult == SQLITE_ROW {
             if Task.isCancelled {
                 if !batch.isEmpty {
                     continuation.yield(.rows(batch))
@@ -168,6 +173,7 @@ actor SQLiteLocalBackend {
                 continuation.yield(.rows(batch))
                 batch.removeAll(keepingCapacity: true)
             }
+            stepResult = sqlite3_step(statement)
         }
 
         if !batch.isEmpty {
@@ -210,18 +216,6 @@ actor SQLiteLocalBackend {
         }
     }
 
-    private func columnNames(of statement: OpaquePointer?, count: Int32) -> [String] {
-        (0..<count).map { index in
-            sqlite3_column_name(statement, index).map { String(cString: $0) } ?? "column_\(index)"
-        }
-    }
-
-    private func columnDeclaredTypes(of statement: OpaquePointer?, count: Int32) -> [String] {
-        (0..<count).map { index in
-            sqlite3_column_decltype(statement, index).map { String(cString: $0) } ?? ""
-        }
-    }
-
     private func rowValues(of statement: OpaquePointer?, count: Int32) -> [PluginCellValue] {
         (0..<count).map { index in
             let colType = sqlite3_column_type(statement, index)

Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift

@@ -268,12 +268,14 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend {
             try bind(parameters, to: statement, db: db)
         }
 
-        let columnCount = sqlite3_column_count(statement)
-        let (columns, columnTypeNames) = Self.columnMetadata(statement, count: columnCount)
+        let firstStep = SQLiteResultColumns.stepFirst(statement)
+        let columnCount = firstStep.count
+        let columns = firstStep.names
+        let columnTypeNames = firstStep.typeNames
 
         var rows: [[PluginCellValue]] = []
         var truncated = false
-        var stepResult = sqlite3_step(statement)
+        var stepResult = firstStep.result
         while stepResult == SQLITE_ROW {
             if rows.count >= PluginRowLimits.emergencyMax {
                 truncated = true
@@ -310,17 +312,17 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend {
             throw SQLitePluginError.queryFailed(String(cString: sqlite3_errmsg(db)))
         }
 
-        let columnCount = sqlite3_column_count(statement)
-        let (columns, columnTypeNames) = Self.columnMetadata(statement, count: columnCount)
+        let firstStep = SQLiteResultColumns.stepFirst(statement)
+        let columnCount = firstStep.count
         continuation.yield(.header(PluginStreamHeader(
-            columns: columns, columnTypeNames: columnTypeNames, estimatedRowCount: nil
+            columns: firstStep.names, columnTypeNames: firstStep.typeNames, estimatedRowCount: nil
         )))
 
         let batchSize = 5_000
         var batch: [PluginRow] = []
         batch.reserveCapacity(batchSize)
 
-        var stepResult = sqlite3_step(statement)
+        var stepResult = firstStep.result
         while stepResult == SQLITE_ROW {
             if Task.isCancelled {
                 if !batch.isEmpty { continuation.yield(.rows(batch)) }
@@ -369,24 +371,6 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend {
         }
     }
 
-    private static func columnMetadata(_ statement: OpaquePointer?, count: Int32) -> ([String], [String]) {
-        var columns: [String] = []
-        var columnTypeNames: [String] = []
-        for i in 0..<count {
-            if let name = sqlite3_column_name(statement, i) {
-                columns.append(String(cString: name))
-            } else {
-                columns.append("column_\(i)")
-            }
-            if let typePtr = sqlite3_column_decltype(statement, i) {
-                columnTypeNames.append(String(cString: typePtr))
-            } else {
-                columnTypeNames.append("")
-            }
-        }
-        return (columns, columnTypeNames)
-    }
-
     private static func readRow(_ statement: OpaquePointer?, columnCount: Int32) -> [PluginCellValue] {
         var row: [PluginCellValue] = []
         for i in 0..<columnCount {

TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift

@@ -72,8 +72,13 @@ extension QueryExecutionCoordinator {
         return tab.tabType == .table ? .tableBrowse : .editor
     }
 
+    /// Never after the table's definition changed: the keys, defaults, generated columns and row
+    /// match policy the tab holds describe the table before the change, and reusing them builds the
+    /// next edit against the old definition.
     func isMetadataCached(tabId: UUID, tableName: String) -> Bool {
-        guard let idx = parent.tabManager.tabs.firstIndex(where: { $0.id == tabId }) else {
+        guard let idx = parent.tabManager.tabs.firstIndex(where: { $0.id == tabId }),
+              !parent.tabSessionRegistry.needsDefinition(tabId)
+        else {
             return false
         }
         let tab = parent.tabManager.tabs[idx]
@@ -168,6 +173,24 @@ extension QueryExecutionCoordinator {
         return resolved
     }
 
+    /// The keys a committed result is edited by. Keys the tab held for the same table carry over
+    /// while its definition is unchanged, until phase 2 reports them. After a definition change they
+    /// describe the table as it was, so they are never carried into the new result.
+    static func resolvedPrimaryKeys(
+        reported: [String]?,
+        engineDefault: String?,
+        previous: [String],
+        definitionChanged: Bool
+    ) -> [String] {
+        if let reported, !reported.isEmpty {
+            return reported
+        }
+        if let engineDefault {
+            return [engineDefault]
+        }
+        return definitionChanged ? [] : previous
+    }
+
     func applyPhase1Result( // swiftlint:disable:this function_parameter_count
         tabId: UUID,
         columns: [String],
@@ -180,6 +203,7 @@ extension QueryExecutionCoordinator {
         isEditable: Bool,
         metadata: ParsedSchemaMetadata?,
         hasSchema: Bool,
+        read: TableFreshness.Read,
         sql: String,
         connection conn: DatabaseConnection,
         isTruncated: Bool = false,
@@ -240,8 +264,13 @@ extension QueryExecutionCoordinator {
             foreignKeysFetched: resolved.foreignKeysFetched
         )
         let previousTableName = parent.tabManager.tabs[idx].tableContext.tableName
+        let definitionChanged = parent.tabSessionRegistry.needsDefinition(existingTabId)
         parent.flushBufferToActiveResult(tabId: existingTabId, pinnedOnly: true)
         parent.setActiveTableRows(newTableRows, for: existingTabId, viewport: viewport)
+        /// A count that started before the change can land after it, while the tab is out of sight,
+        /// and put the old total back. Kept, a total above the automatic-count threshold stops the
+        /// count this read launches, so paging stays bounded by the table as it was.
+        let answeredRowsChange = parent.tabSessionRegistry.recordRead(read, for: existingTabId)
 
         parent.tabManager.mutate(at: idx) { tab in
             tab.schemaVersion += 1
@@ -253,6 +282,9 @@ extension QueryExecutionCoordinator {
             tab.tableContext.isEditable = isEditable
             tab.pagination.isLoading = false
 
+            if answeredRowsChange {
+                tab.pagination.retireDerivedRowCount()
+            }
             if let metadata, let approxCount = metadata.approximateRowCount, approxCount > 0,
                !tab.filterState.hasAppliedFilters {
                 tab.pagination.applyDerivedRowCount(approxCount, isApproximate: true)
@@ -289,16 +321,12 @@ extension QueryExecutionCoordinator {
         }
         parent.toolbarState.isResultsCollapsed = false
 
-        let resolvedPKs: [String]
-        if let pks = metadata?.primaryKeyColumns, !pks.isEmpty {
-            resolvedPKs = pks
-        } else if let defaultPK = PluginManager.shared.defaultPrimaryKeyColumn(for: conn.type) {
-            resolvedPKs = [defaultPK]
-        } else if tableName == previousTableName {
-            resolvedPKs = parent.tabManager.tabs[idx].tableContext.primaryKeyColumns
-        } else {
-            resolvedPKs = []
-        }
+        let resolvedPKs = Self.resolvedPrimaryKeys(
+            reported: metadata?.primaryKeyColumns,
+            engineDefault: PluginManager.shared.defaultPrimaryKeyColumn(for: conn.type),
+            previous: tableName == previousTableName ? parent.tabManager.tabs[idx].tableContext.primaryKeyColumns : [],
+            definitionChanged: definitionChanged
+        )
 
         parent.tabManager.mutate(at: idx) { $0.tableContext.primaryKeyColumns = resolvedPKs }
         captureOrigin(

TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift

@@ -632,6 +632,7 @@ extension QueryExecutionCoordinator {
                 isEditable: isEditable,
                 metadata: inlineMetadata,
                 hasSchema: false,
+                read: TableFreshness.Read(startedAt: claim.startedAt, includesDefinition: false),
                 sql: sql,
                 connection: connection,
                 isTruncated: fetchResult.isTruncated,

TablePro/Core/Coordinators/RowEditingCoordinator+Discard.swift

@@ -64,6 +64,7 @@ extension RowEditingCoordinator {
         if let (_, index) = parent.tabManager.selectedTabAndIndex {
             parent.tabManager.mutate(at: index) { $0.pendingChanges = TabChangeSnapshot() }
         }
+        parent.resumeDeferredTableRefresh()
 
         Task { [parent] in await parent.refreshTables() }
     }

TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift

@@ -3,6 +3,7 @@
 //  TablePro
 //
 
+import Combine
 import Foundation
 import os
 import SwiftUI
@@ -323,11 +324,20 @@ extension RowEditingCoordinator {
         /// The saving tab reloads only when this save changed its rows: rows it wrote, or a truncate of
         /// the table it shows. A save of table operations alone used to re-run whatever the selected
         /// tab held, which on a query tab executed its statement again. A tab on a dropped table closes.
-        guard savingTabIsSelected,
-              let savedTabIndex = parent.tabManager.selectedTabIndex,
-              !tab(at: savedTabIndex, shows: deletedTables),
-              plan.steps.contains(where: { $0.kind == .rowWrite }) || tab(at: savedTabIndex, shows: truncatedTables)
-        else { return }
+        /// A tab another change marked stale while it held these edits reloads now they are saved.
+        let reloadIndex = parent.tabManager.selectedTabIndex.flatMap { index -> Int? in
+            guard savingTabIsSelected, !tab(at: index, shows: deletedTables) else { return nil }
+            let changedItsRows = plan.steps.contains(where: { $0.kind == .rowWrite })
+                || tab(at: index, shows: truncatedTables)
+                || parent.tabSessionRegistry.isStale(parent.tabManager.tabs[index].id)
+            return changedItsRows ? index : nil
+        }
+        announceWrittenTables(
+            plan: plan,
+            truncatedTables: truncatedTables,
+            reloadingTabId: reloadIndex.map { parent.tabManager.tabs[$0].id }
+        )
+        guard let savedTabIndex = reloadIndex else { return }
 
         /// An insert or a delete changes the number this tab is reporting, so a count
         /// the user asked for before the save no longer describes the table. Without
@@ -337,6 +347,35 @@ extension RowEditingCoordinator {
         parent.runQuery(viewport: .keepPlace)
     }
 
+    /// Every table this save wrote rows to or truncated, so a tab showing one in this window or
+    /// another stops showing the rows as they were. The saving tab reloads itself and is left out.
+    private func announceWrittenTables(
+        plan: DataWritePlan,
+        truncatedTables: Set<DatabaseTreeTableRef>,
+        reloadingTabId: UUID?
+    ) {
+        let connectionId = parent.connectionId
+        let adoption = catalogEditAdoption
+        var written = Set(plan.steps.filter { $0.kind == .rowWrite }.compactMap { step in
+            step.tableName.map { DataWriteTarget(database: plan.scope.database, schema: plan.scope.schema, table: $0) }
+        })
+        for ref in truncatedTables {
+            guard let scope = adoption.objectScope(for: ref, connectionId: connectionId) else { continue }
+            written.insert(DataWriteTarget(database: scope.database, schema: scope.schema, table: ref.table.name))
+        }
+        for target in written {
+            AppCommands.shared.objectChanged.send(
+                DatabaseObjectChange(
+                    connectionId: connectionId,
+                    scope: DatabaseScope(connectionId: connectionId, database: target.database, schema: target.schema),
+                    name: target.table,
+                    kind: .rows,
+                    originTabId: reloadingTabId
+                )
+            )
+        }
+    }
+
     /// MySQL, MariaDB and Oracle commit each DROP as it runs, so a save that failed part way can
     /// already have dropped tables its rollback could not bring back. The refreshed catalog says
     /// which, and those are then treated as the drops they were: unstaged, and their tabs closed.

TablePro/Core/Database/DatabaseManager+DocumentWrite.swift

@@ -52,7 +52,9 @@ extension DatabaseManager {
         }
 
         await recordDocumentWrite(statement, scope: scope, databaseType: databaseType, startedAt: startedAt, error: nil)
-        AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: scope.connectionId, scope: scope))
+        AppCommands.shared.objectChanged.send(
+            DatabaseObjectChange(connectionId: scope.connectionId, scope: scope, name: write.table, kind: .rows)
+        )
     }
 
     private func recordDocumentWrite(