From 0bfeb40d61344da2aae2030ec636e59fe7dbcc71 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 23 Sep 2026 22:20:13 +0700 Subject: [PATCH] feat(plugins): list every schema's tables in one query on SQL Server and DuckDB --- CHANGELOG.md | 2 + .../MSSQLSchemaQueries.swift | 45 +++- .../MSSQLSchemaQueriesTests.swift | 34 +++ Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift | 26 +- .../DuckDBSchemaQueries.swift | 52 +++- .../MSSQLPluginDriver+Schema.swift | 44 ++-- .../Plugins/DuckDBSchemaQueriesTests.swift | 30 ++- docs/features/open-quickly.mdx | 9 +- scripts/check-duckdb-offline-metadata.sh | 21 ++ scripts/check-duckdb-table-listing-parity.sh | 200 +++++++++++++++ scripts/check-mssql-table-listing-parity.sh | 241 ++++++++++++++++++ 11 files changed, 646 insertions(+), 58 deletions(-) create mode 100755 scripts/check-duckdb-table-listing-parity.sh create mode 100755 scripts/check-mssql-table-listing-parity.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 36e28b1357..a459e6038a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Show Previous Window Tab** and **Show Next Window Tab** for window tabs, with no default shortcut. - SQLite 3.53.4 built into the SQLite and libSQL drivers in place of the macOS copy. - One-time reset of Open Quickly's Recent query history, and of its objects on connections that switch databases. +- Other-schema tables for Open Quickly and the sidebar filter read in one query on SQL Server. +- Other-schema tables for Open Quickly and the sidebar filter read in one query on DuckDB files. ### Removed diff --git a/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift b/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift index f77cfe1913..207e7c69a0 100644 --- a/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift +++ b/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift @@ -1,5 +1,10 @@ import Foundation +public enum MSSQLTableListingScope: Sendable, Equatable { + case schema(String) + case allSchemas +} + public enum MSSQLSchemaQueries { public static func escape(_ value: String) -> String { MSSQLStringLiteral.escaped(value) @@ -137,7 +142,9 @@ public enum MSSQLSchemaQueries { public static let databases = "SELECT name FROM sys.databases ORDER BY name" - public static let schemas = """ + /// Unordered, because SQL Server rejects an `ORDER BY` in a subquery and the all-schema table + /// listing filters by this query. + internal static let listedSchemaNames = """ SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME NOT IN ( 'information_schema','sys','db_owner','db_accessadmin', @@ -145,17 +152,37 @@ public enum MSSQLSchemaQueries { 'db_datareader','db_datawriter','db_denydatareader', 'db_denydatawriter','guest' ) - ORDER BY SCHEMA_NAME """ + public static let schemas = listedSchemaNames + "\nORDER BY SCHEMA_NAME" + public static func tables(schema: String) -> String { - let s = MSSQLStringLiteral.quoted(schema) + tables(in: .schema(schema)) + } + + /// The same listing over one schema or over every schema `schemas` returns. The second filters + /// by that query itself rather than dropping the schema predicate, so a table is listed here + /// exactly when its schema is listed there, and it projects each row's schema as a third column. + public static func tables(in scope: MSSQLTableListingScope) -> String { + let schemaFilter: String + let schemaColumn: String + let orderBy: String + switch scope { + case .schema(let schema): + schemaFilter = "t.TABLE_SCHEMA = \(MSSQLStringLiteral.quoted(schema))" + schemaColumn = "" + orderBy = "t.TABLE_NAME" + case .allSchemas: + schemaFilter = "t.TABLE_SCHEMA IN (\n\(listedSchemaNames)\n)" + schemaColumn = ", t.TABLE_SCHEMA" + orderBy = "t.TABLE_SCHEMA, t.TABLE_NAME" + } return """ - SELECT t.TABLE_NAME, t.TABLE_TYPE + SELECT t.TABLE_NAME, t.TABLE_TYPE\(schemaColumn) FROM INFORMATION_SCHEMA.TABLES t - WHERE t.TABLE_SCHEMA = \(s) + WHERE \(schemaFilter) AND t.TABLE_TYPE IN ('BASE TABLE', 'VIEW') - ORDER BY t.TABLE_NAME + ORDER BY \(orderBy) """ } @@ -235,10 +262,12 @@ public enum MSSQLSchemaQueries { public struct MSSQLTableRow: Sendable, Equatable { public let name: String public let isView: Bool + public let schema: String? - public init(name: String, isView: Bool) { + public init(name: String, isView: Bool, schema: String? = nil) { self.name = name self.isView = isView + self.schema = schema } } @@ -340,7 +369,7 @@ public extension MSSQLSchemaQueries { static func parseTableRow(_ row: [String?]) -> MSSQLTableRow? { guard let name = row[safe: 0] ?? nil else { return nil } let typeRaw = (row[safe: 1] ?? nil) ?? "BASE TABLE" - return MSSQLTableRow(name: name, isView: typeRaw == "VIEW") + return MSSQLTableRow(name: name, isView: typeRaw == "VIEW", schema: row[safe: 2] ?? nil) } static func parseColumnRow(_ row: [String?]) -> MSSQLColumnRow? { diff --git a/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift b/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift index f5e4003486..991fe7204e 100644 --- a/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift +++ b/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift @@ -49,6 +49,40 @@ final class MSSQLSchemaQueriesTests: XCTestCase { XCTAssertTrue(sql.contains("'VIEW'")) } + func testOneSchemaListingIsFilteredByTheSchemaAlone() { + let sql = MSSQLSchemaQueries.tables(in: .schema("sales")) + XCTAssertEqual(sql, MSSQLSchemaQueries.tables(schema: "sales")) + XCTAssertTrue(sql.contains("WHERE t.TABLE_SCHEMA = N'sales'")) + XCTAssertTrue(sql.hasPrefix("SELECT t.TABLE_NAME, t.TABLE_TYPE\n")) + XCTAssertTrue(sql.hasSuffix("ORDER BY t.TABLE_NAME")) + XCTAssertFalse(sql.contains("INFORMATION_SCHEMA.SCHEMATA")) + } + + /// The all-schema listing is filtered by the schema list query itself, so a table is listed exactly when its schema + /// is one `fetchSchemas()` returns. SQL Server rejects an `ORDER BY` inside that subquery, which is why the list is + /// split from its ordering. + func testAllSchemaListingIsFilteredByTheSchemaListQuery() { + let sql = MSSQLSchemaQueries.tables(in: .allSchemas) + XCTAssertTrue(sql.contains("WHERE t.TABLE_SCHEMA IN (\n\(MSSQLSchemaQueries.listedSchemaNames)\n)")) + XCTAssertFalse(sql.contains("ORDER BY SCHEMA_NAME")) + XCTAssertTrue(sql.hasPrefix("SELECT t.TABLE_NAME, t.TABLE_TYPE, t.TABLE_SCHEMA\n")) + XCTAssertTrue(sql.contains("AND t.TABLE_TYPE IN ('BASE TABLE', 'VIEW')")) + XCTAssertTrue(sql.hasSuffix("ORDER BY t.TABLE_SCHEMA, t.TABLE_NAME")) + } + + func testSchemaListIsTheListedSchemasInOrder() { + XCTAssertEqual(MSSQLSchemaQueries.schemas, MSSQLSchemaQueries.listedSchemaNames + "\nORDER BY SCHEMA_NAME") + XCTAssertFalse(MSSQLSchemaQueries.listedSchemaNames.contains("ORDER BY")) + } + + func testParseTableRowReadsTheSchemaColumnWhenPresent() { + XCTAssertEqual( + MSSQLSchemaQueries.parseTableRow(["orders", "BASE TABLE", "sales"]), + MSSQLTableRow(name: "orders", isView: false, schema: "sales") + ) + XCTAssertNil(MSSQLSchemaQueries.parseTableRow(["orders", "BASE TABLE"])?.schema) + } + func testColumnsQueryIncludesIdentityAndPrimaryKey() { let sql = MSSQLSchemaQueries.columns(schema: "dbo", table: "Users") XCTAssertTrue(sql.contains("IsIdentity")) diff --git a/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift b/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift index ceeb87e8f8..2f3848c6a0 100644 --- a/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift +++ b/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift @@ -582,16 +582,30 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Schema Operations func fetchTables(schema: String?) async throws -> [PluginTableInfo] { - let schemaName = resolveSchema(schema) let result = try await executeParameterized( - query: DuckDBSchemaQueries.listTables, - parameters: [.text(try requireCatalog()), .text(schemaName)] + query: DuckDBSchemaQueries.listTables(in: .schema), + parameters: [.text(try requireCatalog()), .text(resolveSchema(schema))] ) - return result.rows.compactMap { row in + return Self.tableInfos(from: result) + } + + /// A remote catalog answers its schema list best-effort, falling back to `main` when it cannot, + /// and one query filtered by that list has no such fallback, so it keeps the per-schema listing. + func fetchTablesInAllSchemas() async throws -> [PluginTableInfo]? { + guard remoteAlias == nil else { return nil } + let result = try await executeParameterized( + query: DuckDBSchemaQueries.listTables(in: .allSchemas), + parameters: [.text(try requireCatalog())] + ) + return Self.tableInfos(from: result) + } + + private static func tableInfos(from result: PluginQueryResult) -> [PluginTableInfo] { + result.rows.compactMap { row in guard let name = row[safe: 0]?.asText else { return nil } let typeString = (row[safe: 1]?.asText) ?? "BASE TABLE" let tableType = typeString.uppercased().contains("VIEW") ? "VIEW" : "TABLE" - return PluginTableInfo(name: name, type: tableType) + return PluginTableInfo(name: name, type: tableType, schema: row[safe: 2]?.asText) } } @@ -1155,7 +1169,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } else if character == ")" { depth -= 1 if depth == 0 { break } - } else if character == "," , depth == 1 { + } else if character == ",", depth == 1 { keys.append(current) current = "" continue diff --git a/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift b/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift index 56df946795..5e5e87b413 100644 --- a/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift +++ b/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift @@ -9,6 +9,11 @@ import Foundation +enum DuckDBTableListingScope: Sendable, Equatable { + case schema + case allSchemas +} + /// DuckDB's namespace is `catalog.schema.table`, and the `duckdb_*` table functions span /// every attached catalog. A predicate on the schema alone therefore matches same-named /// schemas in other catalogs: with a second database attached, `WHERE schema_name = 'main'` @@ -54,20 +59,39 @@ enum DuckDBSchemaQueries { ORDER BY schema_name """ - static let listTables = """ - SELECT table_name, 'BASE TABLE' AS table_type - FROM duckdb_tables() - WHERE database_name = $1 - AND schema_name = $2 - AND internal = false - UNION ALL - SELECT view_name, 'VIEW' - FROM duckdb_views() - WHERE database_name = $1 - AND schema_name = $2 - AND internal = false - ORDER BY 1 - """ + /// One schema's objects, bound to the catalog and the schema, or every schema's, bound to the + /// catalog alone. The second filters by `listSchemas` itself rather than dropping the schema + /// predicate, so an object is listed here exactly when its schema is listed there, and it + /// projects each row's schema as a third column. + static func listTables(in scope: DuckDBTableListingScope) -> String { + let schemaFilter: String + let schemaColumn: String + let orderBy: String + switch scope { + case .schema: + schemaFilter = "schema_name = $2" + schemaColumn = "" + orderBy = "ORDER BY 1" + case .allSchemas: + schemaFilter = "schema_name IN (\n\(listSchemas)\n)" + schemaColumn = ", schema_name" + orderBy = "ORDER BY 3, 1" + } + return """ + SELECT table_name, 'BASE TABLE' AS table_type\(schemaColumn) + FROM duckdb_tables() + WHERE database_name = $1 + AND \(schemaFilter) + AND internal = false + UNION ALL + SELECT view_name, 'VIEW'\(schemaColumn) + FROM duckdb_views() + WHERE database_name = $1 + AND \(schemaFilter) + AND internal = false + \(orderBy) + """ + } static let columnsForTable = """ SELECT column_name, data_type, is_nullable, column_default, column_index diff --git a/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift b/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift index a56a104f1d..ffdd4549c0 100644 --- a/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift +++ b/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift @@ -14,20 +14,25 @@ extension MSSQLPluginDriver { func fetchTables(schema: String?) async throws -> [PluginTableInfo] { let resolved = effectiveSchema(schema) - let schemaLiteral = MSSQLStringLiteral.quoted(resolved) - let sql = """ - SELECT t.TABLE_NAME, t.TABLE_TYPE - FROM INFORMATION_SCHEMA.TABLES t - WHERE t.TABLE_SCHEMA = \(schemaLiteral) - AND t.TABLE_TYPE IN ('BASE TABLE', 'VIEW') - ORDER BY t.TABLE_NAME - """ - let result = try await execute(query: sql) + return try await listTables(in: .schema(resolved), schemaFallback: resolved) + } + + func fetchTablesInAllSchemas() async throws -> [PluginTableInfo]? { + try await listTables(in: .allSchemas, schemaFallback: nil) + } + + private func listTables( + in scope: MSSQLTableListingScope, + schemaFallback: String? + ) async throws -> [PluginTableInfo] { + let result = try await execute(query: MSSQLSchemaQueries.tables(in: scope)) return result.rows.compactMap { row -> PluginTableInfo? in - guard let name = row[safe: 0]?.asText else { return nil } - let rawType = row[safe: 1]?.asText - let tableType = (rawType == "VIEW") ? "VIEW" : "TABLE" - return PluginTableInfo(name: name, type: tableType, schema: resolved) + guard let table = MSSQLSchemaQueries.parseTableRow(row.map(\.asText)) else { return nil } + return PluginTableInfo( + name: table.name, + type: table.isView ? "VIEW" : "TABLE", + schema: table.schema ?? schemaFallback + ) } } @@ -545,17 +550,7 @@ extension MSSQLPluginDriver { } func fetchSchemas() async throws -> [String] { - let sql = """ - SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA - WHERE SCHEMA_NAME NOT IN ( - 'information_schema','sys','db_owner','db_accessadmin', - 'db_securityadmin','db_ddladmin','db_backupoperator', - 'db_datareader','db_datawriter','db_denydatareader', - 'db_denydatawriter','guest' - ) - ORDER BY SCHEMA_NAME - """ - let result = try await execute(query: sql) + let result = try await execute(query: MSSQLSchemaQueries.schemas) return result.rows.compactMap { $0.first?.asText } } @@ -619,5 +614,4 @@ extension MSSQLPluginDriver { ORDER BY t.name """ } - } diff --git a/TableProTests/Plugins/DuckDBSchemaQueriesTests.swift b/TableProTests/Plugins/DuckDBSchemaQueriesTests.swift index 2a0156aa2d..81d4b11c14 100644 --- a/TableProTests/Plugins/DuckDBSchemaQueriesTests.swift +++ b/TableProTests/Plugins/DuckDBSchemaQueriesTests.swift @@ -22,7 +22,8 @@ import Testing struct DuckDBSchemaQueriesTests { private static let catalogScopedQueries: [(name: String, sql: String)] = [ ("listSchemas", DuckDBSchemaQueries.listSchemas), - ("listTables", DuckDBSchemaQueries.listTables), + ("listTables(in: .schema)", DuckDBSchemaQueries.listTables(in: .schema)), + ("listTables(in: .allSchemas)", DuckDBSchemaQueries.listTables(in: .allSchemas)), ("columnsForTable", DuckDBSchemaQueries.columnsForTable), ("columnsForSchema", DuckDBSchemaQueries.columnsForSchema), ("primaryKeyColumnsForSchema", DuckDBSchemaQueries.primaryKeyColumnsForSchema), @@ -116,9 +117,12 @@ struct DuckDBSchemaQueriesTests { // MARK: - Object listing - @Test("The table list reports views alongside tables and hides internal objects in both") - func tableListIncludesViews() { - let sql = DuckDBSchemaQueries.listTables + @Test( + "The table list reports views alongside tables and hides internal objects in both", + arguments: [DuckDBTableListingScope.schema, .allSchemas] + ) + func tableListIncludesViews(scope: DuckDBTableListingScope) { + let sql = DuckDBSchemaQueries.listTables(in: scope) #expect(sql.contains("duckdb_tables()")) #expect(sql.contains("duckdb_views()")) #expect(sql.contains("'BASE TABLE'")) @@ -129,6 +133,24 @@ struct DuckDBSchemaQueriesTests { ) } + @Test("The one-schema listing binds the schema and projects no schema column") + func oneSchemaListingBindsTheSchema() { + let sql = DuckDBSchemaQueries.listTables(in: .schema) + #expect(sql.components(separatedBy: "schema_name = $2").count == 3) + #expect(!sql.contains(DuckDBSchemaQueries.listSchemas)) + #expect(!sql.contains(", schema_name")) + #expect(sql.hasSuffix("ORDER BY 1")) + } + + @Test("Every arm of the all-schema listing is filtered by the schema list query itself") + func allSchemaListingFiltersByTheSchemaList() { + let sql = DuckDBSchemaQueries.listTables(in: .allSchemas) + #expect(sql.components(separatedBy: "schema_name IN (\n\(DuckDBSchemaQueries.listSchemas)\n)").count == 3) + #expect(!sql.contains("$2")) + #expect(sql.components(separatedBy: ", schema_name\n").count == 3) + #expect(sql.hasSuffix("ORDER BY 3, 1")) + } + // MARK: - Keys @Test("Primary key columns are unnested so a composite key yields one row per column") diff --git a/docs/features/open-quickly.mdx b/docs/features/open-quickly.mdx index e1f4cc424e..fec1e6045d 100644 --- a/docs/features/open-quickly.mdx +++ b/docs/features/open-quickly.mdx @@ -59,7 +59,14 @@ Type a dot to search by where a table lives: | `shop.attendance.timesheet` | The table, when `shop` is the database the connection is browsing | | `"my.schema".orders` | `orders` in a schema whose name holds a dot. Backticks and square brackets quote too | -The browsed schema's tables are listed as the panel opens, and the rest arrive after one catalog query on PostgreSQL or one query per schema on other engines. Until then the panel reads **Loading…** rather than reporting no results. +The browsed schema's tables are listed as the panel opens. Until the rest arrive the panel reads **Loading…** rather than reporting no results. + +| Engine | The other schemas' tables arrive after | +|---|---| +| PostgreSQL, PGlite | One query | +| SQL Server | One query | +| DuckDB | One query, or one per schema on a [remote Quack connection](/databases/duckdb#remote-quack) | +| Every other engine | One query per schema | ## Across connections diff --git a/scripts/check-duckdb-offline-metadata.sh b/scripts/check-duckdb-offline-metadata.sh index 5179879a47..26610fd52a 100755 --- a/scripts/check-duckdb-offline-metadata.sh +++ b/scripts/check-duckdb-offline-metadata.sh @@ -83,6 +83,27 @@ for fragment in "FROM duckdb_tables()" "FROM duckdb_views()" "ORDER BY 2"; do fi done +# The table listing is built for one schema or for every schema rather than declared as a +# constant, so both forms are printed by compiling the query file itself. +cat > "$WORK/builders.swift" <<'SWIFT' +@main +enum Builders { + static func main() { + let queries = [ + ("listTablesInSchema", DuckDBSchemaQueries.listTables(in: .schema)), + ("listTablesInAllSchemas", DuckDBSchemaQueries.listTables(in: .allSchemas)) + ] + for (name, sql) in queries { + print(name) + print(sql) + print("%%") + } + } +} +SWIFT +xcrun swiftc -parse-as-library -module-name Builders "$SWIFT_FILE" "$WORK/builders.swift" -o "$WORK/builders" +"$WORK/builders" >> "$WORK/queries.txt" + cat > "$WORK/probe.c" <<'PROBE' #include #include diff --git a/scripts/check-duckdb-table-listing-parity.sh b/scripts/check-duckdb-table-listing-parity.sh new file mode 100755 index 0000000000..dd663467c6 --- /dev/null +++ b/scripts/check-duckdb-table-listing-parity.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# +# Check that DuckDB's all-schema table listing lists exactly what the per-schema listing does. +# +# Open Quickly and the sidebar filter find tables in schemas nobody has opened through one call, +# DuckDBPluginDriver.fetchTablesInAllSchemas(). Without it the host asks fetchTables(schema:) for +# every schema fetchSchemas() returns. The two have to agree row for row: a table the one-schema +# listing shows and the search cannot find is the bug the listing exists to fix, and one the search +# finds that the sidebar hides is a result that opens nothing. +# +# This compiles the real plugin driver against the shipped static libduckdb, opens a throwaway +# file holding every shape the listing treats specially (views, an empty schema, mixed-case and +# dotted schema and table names, a same-named schema in a second attached catalog, a temporary +# table), and compares the two listings in each catalog, reading every row's schema the way the +# host does. Then it times both listings over a catalog of 300 small schemas. +# +# Usage: +# scripts/check-duckdb-table-listing-parity.sh +# +# Needs Libs/libduckdb.a (scripts/download-libs.sh) and a Debug build of TableProPluginKit in +# DerivedData (run verify.sh build first). Exits non-zero on a disagreement, 3 when a prerequisite +# is missing. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PLUGIN="$ROOT/Plugins/DuckDBDriverPlugin" +LIB="$ROOT/Libs/libduckdb.a" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +[ -f "$LIB" ] || { + echo "$LIB is missing; run scripts/download-libs.sh" >&2 + exit 3 +} + +if [ -z "${DEVELOPER_DIR:-}" ]; then + DEVELOPER_DIR="$(xcode-select -p)" + case "$DEVELOPER_DIR" in + *CommandLineTools*) DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer ;; + esac +fi +export DEVELOPER_DIR + +# The framework this checkout built, so the harness compiles against the PluginKit its sources +# expect; the newest one anywhere is the fallback. +FRAMEWORK_DIR="" +for info in "$HOME"/Library/Developer/Xcode/DerivedData/TablePro-*/info.plist; do + [ -f "$info" ] || continue + workspace="$(/usr/libexec/PlistBuddy -c 'Print :WorkspacePath' "$info" 2> /dev/null)" + products="$(dirname "$info")/Build/Products/Debug" + if [ "$workspace" = "$ROOT/TablePro.xcodeproj" ] && [ -d "$products/TableProPluginKit.framework" ]; then + FRAMEWORK_DIR="$products" + fi +done +if [ -z "$FRAMEWORK_DIR" ]; then + FRAMEWORK_DIR="$(find "$HOME/Library/Developer/Xcode/DerivedData" -type d -path '*/Build/Products/Debug/TableProPluginKit.framework' -print 2> /dev/null \ + | while read -r path; do echo "$(stat -f %m "$path") $(dirname "$path")"; done \ + | sort -rn | head -1 | cut -d' ' -f2-)" +fi +[ -n "$FRAMEWORK_DIR" ] || { + echo "no Debug TableProPluginKit.framework in DerivedData; build the app first" >&2 + exit 3 +} + +cat > "$WORK/main.swift" << 'SWIFT' +import Foundation +import TableProPluginKit + +@main +enum ListingParity { + static func main() async { + let arguments = CommandLine.arguments + let driver = DuckDBPluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: arguments[1] + )) + do { + try await driver.connect() + for statement in arguments[3].components(separatedBy: ";\n") where !statement.isEmpty { + _ = try await driver.execute(query: statement) + } + var failures = 0 + for catalog in [arguments[2], "other"] { + try await driver.switchDatabase(to: catalog) + failures += try await compare(driver, catalog: catalog) + } + try await time(driver, schemaCount: 300) + driver.disconnect() + print(failures == 0 ? "PASS" : "FAIL") + exit(failures == 0 ? 0 : 1) + } catch { + print("error: \(error)") + exit(3) + } + } + + /// Both listings over one catalog of many small schemas, each timed at its best of three. + static func time(_ driver: DuckDBPluginDriver, schemaCount: Int) async throws { + _ = try await driver.execute(query: "ATTACH ':memory:' AS timing") + try await driver.switchDatabase(to: "timing") + for index in 0.. Int64 { + duration.components.seconds * 1_000 + duration.components.attoseconds / 1_000_000_000_000_000 + } + print("\(schemaCount + 1) schemas: per-schema \(milliseconds(perSchema)) ms, all-schema \(milliseconds(allSchemas)) ms") + } + + /// Each per-schema row reads its schema as the host does: the row's own, else the schema asked. + static func compare(_ driver: DuckDBPluginDriver, catalog: String) async throws -> Int { + var perSchema: [String] = [] + for schema in try await driver.fetchSchemas() { + for table in try await driver.fetchTables(schema: schema) { + perSchema.append("\(table.schema ?? schema)|\(table.name)|\(table.type)") + } + } + guard let listed = try await driver.fetchTablesInAllSchemas() else { + print("FAIL \(catalog): fetchTablesInAllSchemas() returned nil") + return 1 + } + let allSchemas = listed.map { "\($0.schema ?? "")|\($0.name)|\($0.type)" } + let missing = Set(perSchema).subtracting(allSchemas).sorted() + let extra = Set(allSchemas).subtracting(perSchema).sorted() + var failures = 0 + if perSchema.count != allSchemas.count || !missing.isEmpty || !extra.isEmpty { + print("FAIL \(catalog): per-schema \(perSchema.count) rows, all-schema \(allSchemas.count) rows") + missing.forEach { print(" only per-schema: \($0)") } + extra.forEach { print(" only all-schema: \($0)") } + failures += 1 + } else { + print("\(catalog): \(allSchemas.count) objects, listings agree") + } + let expected = catalog == "other" + ? ["sales|elsewhere|TABLE"] + : ["Mixed Case|Orders|TABLE", "dot.ted|a.b|TABLE", "main|v_people|VIEW", "sales|orders|TABLE"] + for row in expected where !allSchemas.contains(row) { + print("FAIL \(catalog): the all-schema listing is missing \(row)") + failures += 1 + } + let forbidden = catalog == "other" ? ["sales|orders|TABLE"] : ["sales|elsewhere|TABLE", "main|scratch|TABLE"] + for row in forbidden where allSchemas.contains(row) { + print("FAIL \(catalog): the all-schema listing shows \(row), which belongs to another catalog") + failures += 1 + } + return failures + } +} +SWIFT + +SOURCES=() +while IFS= read -r source; do + SOURCES+=("$source") +done < <(find "$PLUGIN" -maxdepth 1 -name '*.swift' | sort) + +xcrun swiftc -swift-version 6 -parse-as-library -module-name ListingParity -Onone \ + -F "$FRAMEWORK_DIR" -framework TableProPluginKit -Xlinker -rpath -Xlinker "$FRAMEWORK_DIR" \ + -I "$PLUGIN/CDuckDB" -Xcc -I"$PLUGIN/CDuckDB/include" \ + -Xlinker -force_load -Xlinker "$LIB" -lc++ \ + "${SOURCES[@]}" "$WORK/main.swift" -o "$WORK/listing-parity" > "$WORK/compile.log" 2>&1 || { + echo "harness failed to compile:" >&2 + grep -E 'error:' "$WORK/compile.log" | sort -u | head -20 >&2 + exit 3 +} + +FIXTURE="$WORK/fixture.duckdb" +OTHER="$WORK/other.duckdb" +SETUP="CREATE SCHEMA sales; +CREATE SCHEMA \"Mixed Case\"; +CREATE SCHEMA \"dot.ted\"; +CREATE SCHEMA empty_schema; +CREATE TABLE main.people (id INTEGER); +CREATE VIEW main.v_people AS SELECT id FROM main.people; +CREATE TABLE sales.orders (id INTEGER); +CREATE VIEW sales.v_orders AS SELECT id FROM sales.orders; +CREATE TABLE \"Mixed Case\".\"Orders\" (id INTEGER); +CREATE TABLE \"dot.ted\".\"a.b\" (id INTEGER); +CREATE TEMP TABLE scratch (id INTEGER); +ATTACH '$OTHER' AS other; +CREATE SCHEMA other.sales; +CREATE TABLE other.sales.elsewhere (id INTEGER); +CREATE TABLE other.main.lonely (id INTEGER)" + +echo "Checking the all-schema table listing against $(basename "$LIB")" +"$WORK/listing-parity" "$FIXTURE" fixture "$SETUP" diff --git a/scripts/check-mssql-table-listing-parity.sh b/scripts/check-mssql-table-listing-parity.sh new file mode 100755 index 0000000000..474a0a994c --- /dev/null +++ b/scripts/check-mssql-table-listing-parity.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +# +# Check that SQL Server's all-schema table listing lists exactly what the per-schema listing does. +# +# Open Quickly and the sidebar filter find tables in schemas nobody has opened through one query, +# MSSQLSchemaQueries.tables(in: .allSchemas). The sidebar lists each schema through +# tables(in: .schema(name)), for every schema MSSQLSchemaQueries.schemas returns. The two have to +# agree row for row: a table the one-schema listing shows and the search cannot find is the bug +# this listing exists to fix, and one the search finds that the sidebar hides is a result that +# opens nothing. +# +# This builds a database with every shape the listing treats specially (a view, an empty schema, +# mixed-case and dotted names, a schema the reader holds no permission on, tables in the role and +# guest schemas the schema list leaves out), once under the server's default collation and once +# under a case-sensitive one, prints the plugin's real queries from TableProMSSQLCore, and compares +# the two listings as the server administrator and as a login with only SELECT grants. Then it +# times the per-schema listing against the single query over one connection on a database with +# many schemas. +# +# Usage: +# scripts/check-mssql-table-listing-parity.sh [host] [port] [user] [password] +# +# Needs FreeTDS's tsql and a SQL Server the user may create a database and a login on. A throwaway +# server that matches the defaults: +# docker run -d --name listing-mssql -p 1433:1433 -e ACCEPT_EULA=1 -e MSSQL_SA_PASSWORD=Probe_pw1234 \ +# mcr.microsoft.com/azure-sql-edge:latest +# Exits non-zero on a disagreement, 3 when a prerequisite is missing. + +set -uo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-1433}" +ADMIN="${3:-sa}" +ADMIN_PASSWORD="${4:-Probe_pw1234}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# Named per run and dropped only when this run created them, so pointing the check at a shared +# server cannot touch anything that was already there. +RUN_ID="$$_$RANDOM" +READER="tablepro_listing_reader_$RUN_ID" +READER_PASSWORD="Reader_pw_${RUN_ID}_X9" +DATABASES=() +CREATED_READER=0 +WORK="$(mktemp -d)" + +admin_sql() { + printf '%s\ngo\n' "$2" | tsql -H "$HOST" -p "$PORT" -U "$ADMIN" -P "$ADMIN_PASSWORD" -D "$1" -o fhq -t '|' 2>&1 +} + +cleanup() { + local database + for database in "${DATABASES[@]+"${DATABASES[@]}"}"; do + admin_sql master "ALTER DATABASE [$database] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [$database]" > /dev/null + done + if [ "$CREATED_READER" -eq 1 ]; then + admin_sql master "DROP LOGIN [$READER]" > /dev/null + fi + rm -rf "$WORK" +} +trap cleanup EXIT + +command -v tsql > /dev/null || { + echo "tsql not found (brew install freetds)" >&2 + exit 3 +} +if [ -z "${DEVELOPER_DIR:-}" ]; then + DEVELOPER_DIR="$(xcode-select -p)" + case "$DEVELOPER_DIR" in + *CommandLineTools*) DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer ;; + esac +fi +export DEVELOPER_DIR + +if ! admin_sql master "SELECT 1" | grep -qx '1'; then + echo "no SQL Server at $HOST:$PORT as $ADMIN" >&2 + exit 3 +fi + +mkdir -p "$WORK/Sources/ListingSQL" +cat > "$WORK/Package.swift" << EOF +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "ListingSQL", + platforms: [.macOS(.v14)], + dependencies: [.package(path: "$ROOT/Packages/TableProCore")], + targets: [ + .executableTarget( + name: "ListingSQL", + dependencies: [.product(name: "TableProMSSQLCore", package: "TableProCore")] + ) + ] +) +EOF +cat > "$WORK/Sources/ListingSQL/main.swift" << 'SWIFT' +import TableProMSSQLCore + +let arguments = CommandLine.arguments +switch arguments[1] { +case "schemas": + print(MSSQLSchemaQueries.schemas) +case "one": + print(MSSQLSchemaQueries.tables(in: .schema(arguments[2]))) +default: + print(MSSQLSchemaQueries.tables(in: .allSchemas)) +} +SWIFT +swift build --package-path "$WORK" > "$WORK/build.log" 2>&1 || { + echo "the query printer failed to build:" >&2 + grep -E 'error:' "$WORK/build.log" | sort -u | head -20 >&2 + exit 3 +} +PRINTER="$(swift build --package-path "$WORK" --show-bin-path)/ListingSQL" + +VERSION="$(admin_sql master "SELECT CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(64))" | head -1)" +echo "Checking the all-schema table listing against SQL Server $VERSION at $HOST:$PORT" + +admin_sql master "CREATE LOGIN [$READER] WITH PASSWORD = '$READER_PASSWORD', CHECK_POLICY = OFF" | grep -E 'Msg [0-9]+' && exit 3 +CREATED_READER=1 + +# One batch per line: CREATE SCHEMA and CREATE VIEW must each open their own batch. +FIXTURE="CREATE SCHEMA sales +CREATE SCHEMA [Mixed Case] +CREATE SCHEMA [dot.ted] +CREATE SCHEMA empty_schema +CREATE SCHEMA locked +CREATE TABLE dbo.people (id int); CREATE TABLE sales.orders (id int); CREATE TABLE [Mixed Case].[Orders] (id int); CREATE TABLE [dot.ted].[a.b] (id int) +CREATE TABLE locked.secret (id int); CREATE TABLE locked.visible (id int) +CREATE TABLE db_datareader.in_role_schema (id int); CREATE TABLE guest.in_guest_schema (id int) +CREATE VIEW sales.v_orders AS SELECT id FROM sales.orders +CREATE USER [$READER] FOR LOGIN [$READER] +GRANT SELECT ON SCHEMA::dbo TO [$READER]; GRANT SELECT ON SCHEMA::sales TO [$READER]; GRANT SELECT ON SCHEMA::[Mixed Case] TO [$READER]; GRANT SELECT ON SCHEMA::[dot.ted] TO [$READER] +GRANT SELECT ON OBJECT::locked.visible TO [$READER]; GRANT SELECT ON OBJECT::db_datareader.in_role_schema TO [$READER]" + +create_database() { + local database="$1" collation="$2" + admin_sql master "CREATE DATABASE [$database] $collation" | grep -E 'Msg [0-9]+' && return 1 + DATABASES+=("$database") + local line + while IFS= read -r line; do + admin_sql "$database" "$line" | grep -E 'Msg [0-9]+' && return 1 + done <<< "$FIXTURE" + return 0 +} + +# Every row as schema|name|type, in the same shape for both listings. +listings() { + local database="$1" user="$2" password="$3" schema + : > "$WORK/one.txt" + printf '%s\ngo\n' "$("$PRINTER" schemas)" \ + | tsql -H "$HOST" -p "$PORT" -U "$user" -P "$password" -D "$database" -o fhq -t '|' > "$WORK/schemas.txt" 2>&1 + while IFS= read -r schema; do + [ -n "$schema" ] || continue + printf '%s\ngo\n' "$("$PRINTER" one "$schema")" \ + | tsql -H "$HOST" -p "$PORT" -U "$user" -P "$password" -D "$database" -o fhq -t '|' 2>&1 \ + | while IFS='|' read -r name type; do + printf '%s|%s|%s\n' "$schema" "$name" "$type" + done >> "$WORK/one.txt" + done < "$WORK/schemas.txt" + printf '%s\ngo\n' "$("$PRINTER" all)" \ + | tsql -H "$HOST" -p "$PORT" -U "$user" -P "$password" -D "$database" -o fhq -t '|' 2>&1 \ + | while IFS='|' read -r name type schema; do + printf '%s|%s|%s\n' "$schema" "$name" "$type" + done > "$WORK/all.txt" + sort -o "$WORK/one.txt" "$WORK/one.txt" + sort -o "$WORK/all.txt" "$WORK/all.txt" +} + +FAILURES=0 +check() { + local label="$1" database="$2" user="$3" password="$4" + listings "$database" "$user" "$password" + if grep -q 'Msg [0-9]' "$WORK/one.txt" "$WORK/all.txt"; then + echo "FAIL $label: the server rejected a listing query" >&2 + grep -h 'Msg [0-9]' "$WORK/one.txt" "$WORK/all.txt" | sort -u >&2 + FAILURES=$((FAILURES + 1)) + return + fi + if diff -u "$WORK/one.txt" "$WORK/all.txt" > "$WORK/diff.txt"; then + echo "$label: $(wc -l < "$WORK/all.txt" | tr -d ' ') objects, listings agree" + else + echo "FAIL $label: per-schema (-) and all-schema (+) listings differ" >&2 + cat "$WORK/diff.txt" >&2 + FAILURES=$((FAILURES + 1)) + fi + local expected + for expected in "sales|orders|BASE TABLE" "sales|v_orders|VIEW" "Mixed Case|Orders|BASE TABLE" "dot.ted|a.b|BASE TABLE" "locked|visible|BASE TABLE"; do + grep -qxF "$expected" "$WORK/all.txt" || { + echo "FAIL $label: the all-schema listing is missing $expected" >&2 + FAILURES=$((FAILURES + 1)) + } + done + if grep -qE '^(db_datareader|guest)\|' "$WORK/all.txt"; then + echo "FAIL $label: the all-schema listing shows a schema the schema list leaves out" >&2 + FAILURES=$((FAILURES + 1)) + fi +} + +for collation in "" "COLLATE Latin1_General_CS_AS"; do + database="tablepro_listing_parity_${RUN_ID}_${#DATABASES[@]}" + create_database "$database" "$collation" || { + echo "could not build the fixture in $database" >&2 + exit 3 + } + label="${collation:-default collation}" + check "$label, $ADMIN" "$database" "$ADMIN" "$ADMIN_PASSWORD" + check "$label, reader" "$database" "$READER" "$READER_PASSWORD" + if grep -q '^locked|secret|' "$WORK/all.txt"; then + echo "FAIL $label: the reader's all-schema listing shows a table it holds no permission on" >&2 + FAILURES=$((FAILURES + 1)) + fi +done + +# Round trips on one connection: one batch per schema against the single query. +SCHEMA_COUNT=300 +database="tablepro_listing_parity_${RUN_ID}_timing" +admin_sql master "CREATE DATABASE [$database]" | grep -E 'Msg [0-9]+' && exit 3 +DATABASES+=("$database") +{ + for ((i = 0; i < SCHEMA_COUNT; i++)); do + printf 'CREATE SCHEMA s%03d\ngo\nCREATE TABLE s%03d.a (id int); CREATE TABLE s%03d.b (id int)\ngo\n' "$i" "$i" "$i" + done +} | tsql -H "$HOST" -p "$PORT" -U "$ADMIN" -P "$ADMIN_PASSWORD" -D "$database" -o fhq > /dev/null 2>&1 +admin_sql "$database" "$("$PRINTER" schemas)" > "$WORK/timing-schemas.txt" +while IFS= read -r schema; do + printf '%s\ngo\n' "$("$PRINTER" one "$schema")" +done < "$WORK/timing-schemas.txt" > "$WORK/per-schema.sql" +printf '%s\ngo\n' "$("$PRINTER" all)" > "$WORK/all-schemas.sql" +elapsed() { + local start end + start="$(perl -MTime::HiRes=time -e 'printf "%.0f", time * 1000')" + tsql -H "$HOST" -p "$PORT" -U "$ADMIN" -P "$ADMIN_PASSWORD" -D "$database" -o fhq < "$1" > "$2" 2>&1 + end="$(perl -MTime::HiRes=time -e 'printf "%.0f", time * 1000')" + echo $((end - start)) +} +PER_SCHEMA_MS="$(elapsed "$WORK/per-schema.sql" "$WORK/per-schema.out")" +ALL_SCHEMAS_MS="$(elapsed "$WORK/all-schemas.sql" "$WORK/all-schemas.out")" +echo "$(wc -l < "$WORK/timing-schemas.txt" | tr -d ' ') schemas: per-schema $(grep -c . "$WORK/per-schema.out") rows in ${PER_SCHEMA_MS} ms, all-schema $(grep -c . "$WORK/all-schemas.out") rows in ${ALL_SCHEMAS_MS} ms (one connection each)" + +[ "$FAILURES" -eq 0 ] || exit 1 +echo "PASS"